def string_array_builder(client, array, builder):
meta = ObjectMeta()
meta['typename'] = 'vineyard::LargeStringArray'
meta['length_'] = len(array)
meta['null_count_'] = array.null_count
meta['offset_'] = array.offset
null_bitmap = buffer_builder(client, array.buffers()[0], builder)
if isinstance(array, pa.StringArray):
buffer = array.buffers()[1]
length = len(buffer) // (pa.uint32().bit_width // 8)
offset_array = pa.Array.from_buffers(pa.uint32(), length, [None, buffer])
offset_array = offset_array.cast(pa.uint64())
offset_buffer = offset_array.buffers()[1]
else: # is pa.LargeStringArray
offset_buffer = array.buffers()[1]
buffer_offsets = buffer_builder(client, offset_buffer, builder)
buffer_data = buffer_builder(client, array.buffers()[2], builder)
meta.add_member('buffer_offsets_', buffer_offsets)
meta.add_member('buffer_data_', buffer_data)
meta.add_member('null_bitmap_', null_bitmap)
meta['nbytes'] = array.nbytes
return client.create_metadata(meta)
def csc_matrix_builder(client, value, builder, **kw):
meta = ObjectMeta()
meta['typename'] = 'vineyard::CSCMatrix<%s>' % value.dtype.name
meta['value_type_'] = value.dtype.name
meta['shape_'] = to_json(value.shape)
meta['ndim'] = value.ndim
meta['nnz'] = value.nnz
meta.add_member('data', builder.run(client, value.data, **kw))
meta.add_member('indices', builder.run(client, value.indices, **kw))
meta.add_member('indptr', builder.run(client, value.indptr, **kw))
meta['partition_index_'] = to_json(kw.get('partition_index', []))
meta['nbytes'] = value.nnz * value.dtype.itemsize
return client.create_metadata(meta)
def tuple_builder(client, value, builder):
if len(value) == 2:
# use pair
meta = ObjectMeta()
meta['typename'] = 'vineyard::Pair'
meta.add_member('first_', builder.run(client, value[0]))
meta.add_member('second_', builder.run(client, value[1]))
return client.create_metadata(meta)
else:
meta = ObjectMeta()
meta['typename'] = 'vineyard::Tuple'
meta['size_'] = len(value)
for i, item in enumerate(value):
meta.add_member('__elements_-%d' % i, builder.run(client, item))
meta['__elements_-size'] = len(value)
return client.create_metadata(meta)
def mars_sparse_matrix_builder(client, value, builder, **kw):
meta = ObjectMeta()
meta['typename'] = 'vineyard::SparseMatrix<%s>' % value.dtype.name
meta['shape_'] = to_json(value.shape)
meta.add_member('spmatrix', builder.run(client, value.spmatrix, **kw))
return client.create_metadata(meta)
