def _send(self, transfer_type, name: str, tag: str, dst_party: Party, rubbish: Rubbish, table: _DTable, obj=None): tagged_key = f"{name}-{tag}" if transfer_type == federation_pb2.OBJECT: table.put(tagged_key, obj) rubbish.add_obj(table, tagged_key) else: rubbish.add_table(table) data_desc = TransferDataDesc( transferDataType=transfer_type, storageLocator=_get_storage_locator(table), taggedVariableName=_ser_des.serialize(tagged_key)) job = basic_meta_pb2.Job(jobId=self._session_id, name=name) transfer_meta = TransferMeta(job=job, tag=tag, src=self.local_party.to_pb(), dst=dst_party.to_pb(), dataDesc=data_desc, type=federation_pb2.SEND) self._stub.send(transfer_meta)
def async_get(self, name: str, tag: str, parties: list) -> typing.Generator: rubbish = Rubbish(name, tag) futures = self._check_get_status_async(name, tag, parties) for future in as_completed(futures): party = futures[future] obj, head, frags = future.result() if isinstance(obj, _DTable): rubbish.add_table(obj) yield (party, obj) else: table, key = head rubbish.add_obj(table, key) if not is_split_head(obj): yield (party, obj) else: frag_table, frag_keys = frags rubbish.add_table(frag_table) fragments = [frag_table.get(key) for key in frag_keys] yield (party, split_get(fragments)) yield (None, rubbish)
def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Rubbish]: if isinstance(parties, Party): parties = [parties] for party in parties: if (name, tag, party) in _get_tag_histories: raise EnvironmentError(f"get duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) self._get_side_auth(name=name, parties=parties) _status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id) LOGGER.debug(f"[GET] {self._local_party} getting {name}.{tag} from {parties}") tasks = [] for party in parties: _tagged_key = self.__remote__object_key(self._session_id, name, tag, party.role, party.party_id, self._role, self._party_id) tasks.append(check_status_and_get_value(_status_table, _tagged_key)) results = self._loop.run_until_complete(asyncio.gather(*tasks)) rtn = [] rubbish = Rubbish(name, tag) _object_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id) for r in results: LOGGER.debug(f"[GET] {self._local_party} getting {r} from {parties}") if isinstance(r, tuple): _persistent = r[0] == StoreTypes.ROLLPAIR_LMDB table = Standalone.get_instance().table(name=r[1], namespace=r[2], persistent=_persistent, partition=r[3]) rtn.append(table) rubbish.add_table(table) else: # todo: should standalone mode split large object? obj = _object_table.get(r) if obj is None: raise EnvironmentError(f"federation get None from {parties} with name {name}, tag {tag}") rtn.append(obj) rubbish.add_obj(_object_table, r) rubbish.add_obj(_status_table, r) return rtn, rubbish
def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Rubbish: if obj is None: raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}") if isinstance(parties, Party): parties = [parties] for party in parties: if (name, tag, party) in _remote_tag_histories: raise EnvironmentError(f"remote duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) self._remote_side_auth(name=name, parties=parties) rubbish = Rubbish(name, tag) for party in parties: _tagged_key = self.__remote__object_key(self._session_id, name, tag, self._role, self._party_id, party.role, party.party_id) _status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id) if isinstance(obj, _DTable): obj.set_gc_disable() # noinspection PyProtectedMember _status_table.put(_tagged_key, (obj._type, obj._name, obj._namespace, obj._partitions)) rubbish.add_table(obj) rubbish.add_obj(_status_table, _tagged_key) else: _table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id) _table.put(_tagged_key, obj) _status_table.put(_tagged_key, _tagged_key) rubbish.add_obj(_table, _tagged_key) rubbish.add_obj(_status_table, _tagged_key) LOGGER.debug("[REMOTE] Sent {}".format(_tagged_key)) return rubbish