def string_array_builder(client, array, builder): meta = ObjectMeta() meta['typename'] = 'vineyard::LargeStringArray' meta['length_'] = len(array) meta['null_count_'] = array.null_count meta['offset_'] = array.offset null_bitmap = buffer_builder(client, array.buffers()[0], builder) if isinstance(array, pa.StringArray): buffer = array.buffers()[1] length = len(buffer) // (pa.uint32().bit_width // 8) offset_array = pa.Array.from_buffers(pa.uint32(), length, [None, buffer]) offset_array = offset_array.cast(pa.uint64()) offset_buffer = offset_array.buffers()[1] else: # is pa.LargeStringArray offset_buffer = array.buffers()[1] buffer_offsets = buffer_builder(client, offset_buffer, builder) buffer_data = buffer_builder(client, array.buffers()[2], builder) meta.add_member('buffer_offsets_', buffer_offsets) meta.add_member('buffer_data_', buffer_data) meta.add_member('null_bitmap_', null_bitmap) meta['nbytes'] = array.nbytes return client.create_metadata(meta)
def csc_matrix_builder(client, value, builder, **kw): meta = ObjectMeta() meta['typename'] = 'vineyard::CSCMatrix<%s>' % value.dtype.name meta['value_type_'] = value.dtype.name meta['shape_'] = to_json(value.shape) meta['ndim'] = value.ndim meta['nnz'] = value.nnz meta.add_member('data', builder.run(client, value.data, **kw)) meta.add_member('indices', builder.run(client, value.indices, **kw)) meta.add_member('indptr', builder.run(client, value.indptr, **kw)) meta['partition_index_'] = to_json(kw.get('partition_index', [])) meta['nbytes'] = value.nnz * value.dtype.itemsize return client.create_metadata(meta)
def tuple_builder(client, value, builder): if len(value) == 2: # use pair meta = ObjectMeta() meta['typename'] = 'vineyard::Pair' meta.add_member('first_', builder.run(client, value[0])) meta.add_member('second_', builder.run(client, value[1])) return client.create_metadata(meta) else: meta = ObjectMeta() meta['typename'] = 'vineyard::Tuple' meta['size_'] = len(value) for i, item in enumerate(value): meta.add_member('__elements_-%d' % i, builder.run(client, item)) meta['__elements_-size'] = len(value) return client.create_metadata(meta)
def mars_sparse_matrix_builder(client, value, builder, **kw): meta = ObjectMeta() meta['typename'] = 'vineyard::SparseMatrix<%s>' % value.dtype.name meta['shape_'] = to_json(value.shape) meta.add_member('spmatrix', builder.run(client, value.spmatrix, **kw)) return client.create_metadata(meta)