def _send(self,
transfer_type,
name: str,
tag: str,
dst_party: Party,
rubbish: Rubbish,
table: _DTable,
obj=None):
tagged_key = f"{name}-{tag}"
if transfer_type == federation_pb2.OBJECT:
table.put(tagged_key, obj)
rubbish.add_obj(table, tagged_key)
else:
rubbish.add_table(table)
data_desc = TransferDataDesc(
transferDataType=transfer_type,
storageLocator=_get_storage_locator(table),
taggedVariableName=_ser_des.serialize(tagged_key))
job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
transfer_meta = TransferMeta(job=job,
tag=tag,
src=self.local_party.to_pb(),
dst=dst_party.to_pb(),
dataDesc=data_desc,
type=federation_pb2.SEND)
self._stub.send(transfer_meta)
def async_get(self, name: str, tag: str,
parties: list) -> typing.Generator:
rubbish = Rubbish(name, tag)
futures = self._check_get_status_async(name, tag, parties)
for future in as_completed(futures):
party = futures[future]
obj, head, frags = future.result()
if isinstance(obj, _DTable):
rubbish.add_table(obj)
yield (party, obj)
else:
table, key = head
rubbish.add_obj(table, key)
if not is_split_head(obj):
yield (party, obj)
else:
frag_table, frag_keys = frags
rubbish.add_table(frag_table)
fragments = [frag_table.get(key) for key in frag_keys]
yield (party, split_get(fragments))
yield (None, rubbish)
def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Rubbish]:
if isinstance(parties, Party):
parties = [parties]
for party in parties:
if (name, tag, party) in _get_tag_histories:
raise EnvironmentError(f"get duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
self._get_side_auth(name=name, parties=parties)
_status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id)
LOGGER.debug(f"[GET] {self._local_party} getting {name}.{tag} from {parties}")
tasks = []
for party in parties:
_tagged_key = self.__remote__object_key(self._session_id, name, tag, party.role, party.party_id, self._role,
self._party_id)
tasks.append(check_status_and_get_value(_status_table, _tagged_key))
results = self._loop.run_until_complete(asyncio.gather(*tasks))
rtn = []
rubbish = Rubbish(name, tag)
_object_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id)
for r in results:
LOGGER.debug(f"[GET] {self._local_party} getting {r} from {parties}")
if isinstance(r, tuple):
_persistent = r[0] == StoreTypes.ROLLPAIR_LMDB
table = Standalone.get_instance().table(name=r[1], namespace=r[2], persistent=_persistent,
partition=r[3])
rtn.append(table)
rubbish.add_table(table)
else: # todo: should standalone mode split large object?
obj = _object_table.get(r)
if obj is None:
raise EnvironmentError(f"federation get None from {parties} with name {name}, tag {tag}")
rtn.append(obj)
rubbish.add_obj(_object_table, r)
rubbish.add_obj(_status_table, r)
return rtn, rubbish
def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Rubbish:
if obj is None:
raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}")
if isinstance(parties, Party):
parties = [parties]
for party in parties:
if (name, tag, party) in _remote_tag_histories:
raise EnvironmentError(f"remote duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
self._remote_side_auth(name=name, parties=parties)
rubbish = Rubbish(name, tag)
for party in parties:
_tagged_key = self.__remote__object_key(self._session_id, name, tag, self._role, self._party_id, party.role,
party.party_id)
_status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id)
if isinstance(obj, _DTable):
obj.set_gc_disable()
# noinspection PyProtectedMember
_status_table.put(_tagged_key, (obj._type, obj._name, obj._namespace, obj._partitions))
rubbish.add_table(obj)
rubbish.add_obj(_status_table, _tagged_key)
else:
_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id)
_table.put(_tagged_key, obj)
_status_table.put(_tagged_key, _tagged_key)
rubbish.add_obj(_table, _tagged_key)
rubbish.add_obj(_status_table, _tagged_key)
LOGGER.debug("[REMOTE] Sent {}".format(_tagged_key))
return rubbish