def create_global_dataframe(vineyard_endpoint: str, results: List[List[ObjectID]], name: str, **kwargs) -> ObjectID: # use the partial_id_matrix and the name in **kwargs to create a global # dataframe. # # Here the `name`` is given in the the input URI path in the form of # vineyard://{name_for_the_global_dataframe} if name is None: raise ValueError("Name of the global dataframe is not provided") chunks = [] for subresults in results: chunks.extend(subresults) vineyard_rpc_client = vineyard.connect(vineyard_endpoint) extra_meta = { 'partition_shape_row_': len(results), 'partition_shape_column_': 1, 'nbytes': 0, # FIXME } gdf = make_global_dataframe(vineyard_rpc_client, chunks, extra_meta) vineyard_rpc_client.put_name(gdf, name) vineyard_rpc_client.persist(gdf.id) return gdf.id
def execute(cls, ctx, op): if vineyard is None: raise RuntimeError("vineyard is not available") socket, _ = resolve_vineyard_socket(ctx, op) client = vineyard.connect(socket) # # store the result object id to execution context chunks = [ctx[chunk.key][0][0] for chunk in op.inputs] ctx[op.outputs[0].key] = pd.DataFrame( {0: [make_global_dataframe(client, chunks).id]})
def test_dask_dataframe_resolver(dask_cluster): clients, dask_scheduler, dask_workers = dask_cluster chunks = [] for i, client in enumerate(clients): chunk = client.put(pd.DataFrame({'x': [i, i * 2], 'y': [i * 3, i * 4]})) client.persist(chunk) chunks.append(chunk) gdf = make_global_dataframe(clients[0], chunks) ddf = clients[0].get(gdf.id, dask_scheduler=dask_scheduler, dask_workers=dask_workers) assert ddf.sum().sum().compute() == 60
def dask_dataframe_builder(client, value, builder, **kw): def put_partition(v, partition_info=None): client = vineyard.connect() obj_id = client.put(v, partition_index=(partition_info['number'], 0)) client.persist(obj_id) return pd.DataFrame([{ 'no': partition_info['number'], 'id': int(obj_id) }]) _ = Client(kw['dask_scheduler']) # enforce distributed scheduling res = value.map_partitions(put_partition, meta={ 'no': int, 'id': int }).compute() res = res.set_index('no') blocks = [res.loc[i] for i in range(len(res))] return make_global_dataframe(client, blocks)
def create_global_dataframe(self, results, **kwargs): # use the partial_id_matrix and the name in **kwargs # to create a global dataframe. Here the name is given in the # the input URI path in the # form of vineyard://{name_for_the_global_dataframe} name = kwargs.pop("name", None) if name is None: raise ValueError("Name of the global dataframe is not provided") chunks = [] for row in results: for chunk in row: chunks.append(chunk) vineyard_rpc_client = vineyard.connect(self.vineyard_endpoint) extra_meta = { 'partition_shape_row_': len(results), 'partition_shape_column_': 1, 'nbytes': 0, # FIXME } gdf = make_global_dataframe(vineyard_rpc_client, chunks, extra_meta) vineyard_rpc_client.put_name(gdf, name)
def to_vineyard(self): client = vineyard.connect() block_to_vineyard = cached_remote_fn(_block_to_vineyard, num_cpus=0.1) blocks = ray.get( [block_to_vineyard.remote(block) for block in self._blocks]) return make_global_dataframe(client, blocks).id