def tf_tensor_resolver(obj): meta = obj.meta num = from_json(meta['num']) data_shape = from_json(meta['data_shape_']) label_shape = from_json(meta['label_shape_']) data_name = meta['data_type_'] label_name = meta['label_type_'] data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None)) label_type = normalize_dtype(label_name, meta.get('value_type_meta_', None)) data = np.frombuffer(memoryview(obj.member('buffer_data_')), dtype=data_type).reshape(data_shape) label = np.frombuffer(memoryview(obj.member('buffer_label_')), dtype=label_type).reshape(label_shape) data = tf.data.Dataset.from_tensor_slices((data, label)) return data
def mxnet_tensor_resolver(obj, resolver, **kw): meta = obj.meta data_shape = from_json(meta['data_shape_']) label_shape = from_json(meta['label_shape_']) data_name = meta['data_type_'] label_name = meta['label_type_'] data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None)) label_type = normalize_dtype(label_name, meta.get('value_type_meta_', None)) data = np.frombuffer(memoryview(obj.member('buffer_data_')), dtype=data_type).reshape(data_shape) label = np.frombuffer(memoryview(obj.member('buffer_label_')), dtype=label_type).reshape(label_shape) return mx.gluon.data.ArrayDataset((data, label))
def torch_tensor_resolver(obj): meta = obj.meta data_shape = from_json(meta['data_shape_']) label_shape = from_json(meta['label_shape_']) data_name = meta['data_type_'] label_name = meta['label_type_'] data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None)) label_type = normalize_dtype(label_name, meta.get('value_type_meta_', None)) data = torch.from_numpy( np.frombuffer(memoryview(obj.member('buffer_data_')), dtype=data_type).reshape(data_shape)) label = torch.from_numpy( np.frombuffer(memoryview(obj.member('buffer_label_')), dtype=label_type).reshape(label_shape)) return torch.utils.data.TensorDataset(data, label)
def dali_tensor_resolver(obj, **kw): assert dali is not None, "Nvidia DALI is not available" meta = obj.meta data_shape = from_json(meta['data_shape_']) label_shape = from_json(meta['label_shape_']) data_name = meta['data_type_'] label_name = meta['label_type_'] data_type = normalize_dtype(data_name, meta.get('value_type_meta_', None)) label_type = normalize_dtype(label_name, meta.get('value_type_meta_', None)) data = np.frombuffer(memoryview(obj.member('buffer_data_')), dtype=data_type).reshape(data_shape) label = np.frombuffer(memoryview(obj.member('buffer_label_')), dtype=label_type).reshape(label_shape) pipe_out = dali_pipe(data, label, device_id=device_id, num_threads=num_threads, batch_size=batch_size) pipe_out.build() pipe_output = pipe_out.run() return pipe_output
def execute(cls, ctx, op): if vineyard is None: raise RuntimeError("vineyard is not available") socket = resolve_vineyard_socket(ctx, op) client = vineyard.connect(socket) meta = client.get_meta(vineyard.ObjectID(op.object_id)) chunks, dtypes = [], None for idx in range(meta["partitions_-size"]): chunk_meta = meta["partitions_-%d" % idx] columns = pd.Index(from_json(chunk_meta["columns_"])) shape = (np.nan, len(columns)) if not chunk_meta.islocal: continue if dtypes is None: dtypes = [] for idx in range(len(columns)): column_meta = chunk_meta["__values_-value-%d" % idx] dtype = normalize_dtype( column_meta["value_type_"], column_meta.get("value_type_meta_", None), ) dtypes.append(dtype) dtypes = pd.Series(dtypes, index=columns) chunk_index = ( chunk_meta["partition_index_row_"], chunk_meta["partition_index_column_"], ) # chunk: (chunk_id, worker_address, dtype, shape, index, columns) chunks.append( ( repr(chunk_meta.id), ctx.worker_address, dtypes, shape, chunk_index, columns, ) ) ctx[op.outputs[0].key] = pd.DataFrame(chunks, columns=cls.generated_columns)
def mars_sparse_matrix_resolver(obj, resolver) -> sparse.SparseNDArray: meta = obj.meta shape = from_json(meta['shape_']) spmatrix = resolver.run(obj.member('spmatrix')) return sparse.matrix.SparseMatrix(spmatrix, shape=shape)