def load(self, address: AddressABC, partitions: int, schema: dict, **kwargs): from fate_arch.common.address import EggRollAddress from fate_arch.storage import EggRollStorageType if isinstance(address, EggRollAddress): options = kwargs.get("option", {}) options["total_partitions"] = partitions options["store_type"] = address.storage_type options["create_if_missing"] = False rp = self._rpc.load(namespace=address.namespace, name=address.name, options=options) if rp is None or rp.get_partitions() == 0: raise RuntimeError(f"no exists: {address.name}, {address.namespace}, {address.storage_type}") if address.storage_type != EggRollStorageType.ROLLPAIR_IN_MEMORY: rp = rp.save_as(name=f"{address.name}_{fate_uuid()}", namespace=self.session_id, partition=partitions, options={'store_type': EggRollStorageType.ROLLPAIR_IN_MEMORY}) table = Table(rp=rp) table.schema = schema return table from fate_arch.common.address import FileAddress if isinstance(address, FileAddress): return address raise NotImplementedError(f"address type {type(address)} not supported with eggroll backend")
def parallelize(self, data, partition: int, include_key: bool, **kwargs) -> Table: options = dict() options["total_partitions"] = partition options["include_key"] = include_key rp = self._rpc.parallelize(data=data, options=options) return Table(rp)
def load(self, address: AddressABC, partitions: int, schema: dict, **kwargs): from fate_arch.common.address import EggRollAddress from fate_arch.storage import EggRollStoreType if isinstance(address, EggRollAddress): options = kwargs.get("option", {}) options["total_partitions"] = partitions options["store_type"] = kwargs.get("store_type", EggRollStoreType.ROLLPAIR_LMDB) options["create_if_missing"] = False rp = self._rpc.load(namespace=address.namespace, name=address.name, options=options) if rp is None or rp.get_partitions() == 0: raise RuntimeError( f"no exists: {address.name}, {address.namespace}") if options["store_type"] != EggRollStoreType.ROLLPAIR_IN_MEMORY: rp = rp.save_as( name=f"{address.name}_{fate_uuid()}", namespace=self.session_id, partition=partitions, options={ "store_type": EggRollStoreType.ROLLPAIR_IN_MEMORY }, ) table = Table(rp=rp) table.schema = schema return table from fate_arch.common.address import PathAddress if isinstance(address, PathAddress): from fate_arch.computing.non_distributed import LocalData from fate_arch.computing import ComputingEngine return LocalData(address.path, engine=ComputingEngine.EGGROLL) raise NotImplementedError( f"address type {type(address)} not supported with eggroll backend")
def get(self, name, tag, parties, gc): parties = [(party.role, party.party_id) for party in parties] raw_result = _get(name, tag, parties, self._rsc, gc) return [Table(v) if isinstance(v, RollPair) else v for v in raw_result]