def combine_missing(a, b): # return a copy of a with missing values of a and b combined if a.null_count > 0 or b.null_count > 0: a, b = vaex.arrow.convert.align(a, b) if isinstance(a, pa.ChunkedArray): # divide and conquer assert isinstance(b, pa.ChunkedArray) assert len(a.chunks) == len(b.chunks) return pa.chunked_array([ combine_missing(ca, cb) for ca, cb in zip(a.chunks, b.chunks) ]) if a.offset != 0: a = vaex.arrow.convert.trim_buffers_ipc(a) if b.offset != 0: b = vaex.arrow.convert.trim_buffers_ipc(b) assert a.offset == 0 assert b.offset == 0 # not optimal nulls = pc.invert(pc.or_(a.is_null(), b.is_null())) assert nulls.offset == 0 nulls_buffer = nulls.buffers()[1] # this is not the case: no reason why it should be (TODO: open arrow issue) # assert nulls.buffers()[0] is None buffers = a.buffers() return pa.Array.from_buffers(a.type, len(a), [nulls_buffer] + buffers[1:]) else: return a
def test_logical(): a = pa.array([True, False, False, None]) b = pa.array([True, True, False, True]) assert pc.and_(a, b) == pa.array([True, False, False, None]) assert pc.and_kleene(a, b) == pa.array([True, False, False, None]) assert pc.or_(a, b) == pa.array([True, True, False, None]) assert pc.or_kleene(a, b) == pa.array([True, True, False, True]) assert pc.xor(a, b) == pa.array([False, True, False, None]) assert pc.invert(a) == pa.array([False, True, True, None])
def combine_missing(a, b): assert a.offset == 0 if a.null_count > 0 or b.null_count > 0: # not optimal nulls = pc.invert(pc.or_(a.is_null(), b.is_null())) assert nulls.offset == 0 nulls_buffer = nulls.buffers()[1] # this is not the case: no reason why it should be (TODO: open arrow issue) # assert nulls.buffers()[0] is None else: nulls_buffer = None buffers = a.buffers() return pa.Array.from_buffers(a.type, len(a), [nulls_buffer, buffers[1]])
def binary_col(op, l, r): """ interpretor for executing binary operator expressions """ if op == "+": return compute.add_checked(l, r) if op == "*": return compute.multiply_checked(l, r) if op == '-': return compute.subtract_checked(l, r) if op == "=": return compute.equal(l, r) if op == "<>": return compute.not_equal(l, r) if op == "!=": return compute.not_equal(l, r) if op == "or": return compute.or_(l, r) if op == "<": return compute.less(l, r) if op == ">": return compute.greater(l, r) if op == "/": return compute.divide_checked(l, r) if op == "and": return compute.and_(l, r) if op == "in": return compute.is_in(l, r) if op == "==": return compute.equal(l, r) if op == "<=": return compute.less_equal(l, r) if op == ">=": return compute.greater_equal(l, r) raise Exception("binary op not implemented")