def tf_tensor_resolver(obj):
meta = obj.meta
num = from_json(meta['num'])
data_shape = from_json(meta['data_shape_'])
label_shape = from_json(meta['label_shape_'])
data_name = meta['data_type_']
label_name = meta['label_type_']
data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None))
label_type = normalize_dtype(label_name, meta.get('value_type_meta_', None))
data = np.frombuffer(memoryview(obj.member('buffer_data_')), dtype=data_type).reshape(data_shape)
label = np.frombuffer(memoryview(obj.member('buffer_label_')), dtype=label_type).reshape(label_shape)
data = tf.data.Dataset.from_tensor_slices((data, label))
return data
def mxnet_tensor_resolver(obj, resolver, **kw):
meta = obj.meta
data_shape = from_json(meta['data_shape_'])
label_shape = from_json(meta['label_shape_'])
data_name = meta['data_type_']
label_name = meta['label_type_']
data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None))
label_type = normalize_dtype(label_name, meta.get('value_type_meta_',
None))
data = np.frombuffer(memoryview(obj.member('buffer_data_')),
dtype=data_type).reshape(data_shape)
label = np.frombuffer(memoryview(obj.member('buffer_label_')),
dtype=label_type).reshape(label_shape)
return mx.gluon.data.ArrayDataset((data, label))
def torch_tensor_resolver(obj):
meta = obj.meta
data_shape = from_json(meta['data_shape_'])
label_shape = from_json(meta['label_shape_'])
data_name = meta['data_type_']
label_name = meta['label_type_']
data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None))
label_type = normalize_dtype(label_name, meta.get('value_type_meta_',
None))
data = torch.from_numpy(
np.frombuffer(memoryview(obj.member('buffer_data_')),
dtype=data_type).reshape(data_shape))
label = torch.from_numpy(
np.frombuffer(memoryview(obj.member('buffer_label_')),
dtype=label_type).reshape(label_shape))
return torch.utils.data.TensorDataset(data, label)
def dali_tensor_resolver(obj, **kw):
assert dali is not None, "Nvidia DALI is not available"
meta = obj.meta
data_shape = from_json(meta['data_shape_'])
label_shape = from_json(meta['label_shape_'])
data_name = meta['data_type_']
label_name = meta['label_type_']
data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None))
label_type = normalize_dtype(label_name, meta.get('value_type_meta_',
None))
data = np.frombuffer(memoryview(obj.member('buffer_data_')),
dtype=data_type).reshape(data_shape)
label = np.frombuffer(memoryview(obj.member('buffer_label_')),
dtype=label_type).reshape(label_shape)
pipe_out = dali_pipe(data,
label,
device_id=device_id,
num_threads=num_threads,
batch_size=batch_size)
pipe_out.build()
pipe_output = pipe_out.run()
return pipe_output
def execute(cls, ctx, op):
if vineyard is None:
raise RuntimeError("vineyard is not available")
socket = resolve_vineyard_socket(ctx, op)
client = vineyard.connect(socket)
meta = client.get_meta(vineyard.ObjectID(op.object_id))
chunks, dtypes = [], None
for idx in range(meta["partitions_-size"]):
chunk_meta = meta["partitions_-%d" % idx]
columns = pd.Index(from_json(chunk_meta["columns_"]))
shape = (np.nan, len(columns))
if not chunk_meta.islocal:
continue
if dtypes is None:
dtypes = []
for idx in range(len(columns)):
column_meta = chunk_meta["__values_-value-%d" % idx]
dtype = normalize_dtype(
column_meta["value_type_"],
column_meta.get("value_type_meta_", None),
)
dtypes.append(dtype)
dtypes = pd.Series(dtypes, index=columns)
chunk_index = (
chunk_meta["partition_index_row_"],
chunk_meta["partition_index_column_"],
)
# chunk: (chunk_id, worker_address, dtype, shape, index, columns)
chunks.append(
(
repr(chunk_meta.id),
ctx.worker_address,
dtypes,
shape,
chunk_index,
columns,
)
)
ctx[op.outputs[0].key] = pd.DataFrame(chunks, columns=cls.generated_columns)
def mars_sparse_matrix_resolver(obj, resolver) -> sparse.SparseNDArray:
meta = obj.meta
shape = from_json(meta['shape_'])
spmatrix = resolver.run(obj.member('spmatrix'))
return sparse.matrix.SparseMatrix(spmatrix, shape=shape)
