def load(self, address: AddressABC, partitions: int, schema: dict, **kwargs):
from fate_arch.common.address import EggRollAddress
from fate_arch.storage import EggRollStorageType
if isinstance(address, EggRollAddress):
options = kwargs.get("option", {})
options["total_partitions"] = partitions
options["store_type"] = address.storage_type
options["create_if_missing"] = False
rp = self._rpc.load(namespace=address.namespace, name=address.name, options=options)
if rp is None or rp.get_partitions() == 0:
raise RuntimeError(f"no exists: {address.name}, {address.namespace}, {address.storage_type}")
if address.storage_type != EggRollStorageType.ROLLPAIR_IN_MEMORY:
rp = rp.save_as(name=f"{address.name}_{fate_uuid()}", namespace=self.session_id, partition=partitions,
options={'store_type': EggRollStorageType.ROLLPAIR_IN_MEMORY})
table = Table(rp=rp)
table.schema = schema
return table
from fate_arch.common.address import FileAddress
if isinstance(address, FileAddress):
return address
raise NotImplementedError(f"address type {type(address)} not supported with eggroll backend")
def parallelize(self, data, partition: int, include_key: bool,
**kwargs) -> Table:
options = dict()
options["total_partitions"] = partition
options["include_key"] = include_key
rp = self._rpc.parallelize(data=data, options=options)
return Table(rp)
def load(self, address: AddressABC, partitions: int, schema: dict,
**kwargs):
from fate_arch.common.address import EggRollAddress
from fate_arch.storage import EggRollStoreType
if isinstance(address, EggRollAddress):
options = kwargs.get("option", {})
options["total_partitions"] = partitions
options["store_type"] = kwargs.get("store_type",
EggRollStoreType.ROLLPAIR_LMDB)
options["create_if_missing"] = False
rp = self._rpc.load(namespace=address.namespace,
name=address.name,
options=options)
if rp is None or rp.get_partitions() == 0:
raise RuntimeError(
f"no exists: {address.name}, {address.namespace}")
if options["store_type"] != EggRollStoreType.ROLLPAIR_IN_MEMORY:
rp = rp.save_as(
name=f"{address.name}_{fate_uuid()}",
namespace=self.session_id,
partition=partitions,
options={
"store_type": EggRollStoreType.ROLLPAIR_IN_MEMORY
},
)
table = Table(rp=rp)
table.schema = schema
return table
from fate_arch.common.address import PathAddress
if isinstance(address, PathAddress):
from fate_arch.computing.non_distributed import LocalData
from fate_arch.computing import ComputingEngine
return LocalData(address.path, engine=ComputingEngine.EGGROLL)
raise NotImplementedError(
f"address type {type(address)} not supported with eggroll backend")
def get(self, name, tag, parties, gc):
parties = [(party.role, party.party_id) for party in parties]
raw_result = _get(name, tag, parties, self._rsc, gc)
return [Table(v) if isinstance(v, RollPair) else v for v in raw_result]