def _send(self,
transfer_type,
name: str,
tag: str,
dst_party: Party,
rubbish: Rubbish,
table: _DTable,
obj=None):
tagged_key = f"{name}-{tag}"
if transfer_type == federation_pb2.OBJECT:
table.put(tagged_key, obj)
rubbish.add_obj(table, tagged_key)
else:
rubbish.add_table(table)
data_desc = TransferDataDesc(
transferDataType=transfer_type,
storageLocator=_get_storage_locator(table),
taggedVariableName=_ser_des.serialize(tagged_key))
job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
transfer_meta = TransferMeta(job=job,
tag=tag,
src=self.local_party.to_pb(),
dst=dst_party.to_pb(),
dataDesc=data_desc,
type=federation_pb2.SEND)
self._stub.send(transfer_meta)
def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Rubbish:
if obj is None:
raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}")
if isinstance(parties, Party):
parties = [parties]
for party in parties:
if (name, tag, party) in _remote_tag_histories:
raise EnvironmentError(f"remote duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
self._remote_side_auth(name=name, parties=parties)
rubbish = Rubbish(name, tag)
for party in parties:
_tagged_key = self.__remote__object_key(self._session_id, name, tag, self._role, self._party_id, party.role,
party.party_id)
_status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id)
if isinstance(obj, _DTable):
obj.set_gc_disable()
# noinspection PyProtectedMember
_status_table.put(_tagged_key, (obj._type, obj._name, obj._namespace, obj._partitions))
rubbish.add_table(obj)
rubbish.add_obj(_status_table, _tagged_key)
else:
_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id)
_table.put(_tagged_key, obj)
_status_table.put(_tagged_key, _tagged_key)
rubbish.add_obj(_table, _tagged_key)
rubbish.add_obj(_status_table, _tagged_key)
LOGGER.debug("[REMOTE] Sent {}".format(_tagged_key))
return rubbish
def get(self, name, tag, parties: Union[Party, list]):
rs = self.rsc.load(name=name, tag=tag)
rubbish = Rubbish(name, tag)
if isinstance(parties, Party):
parties = [parties]
rs_parties = [(party.role, party.party_id) for party in parties]
for party in parties:
if (name, tag, party) in _get_tag_histories:
raise EnvironmentError(f"get duplicate tag {(name, tag)}")
_get_tag_histories.add((name, tag, party))
# TODO:0: check if exceptions are swallowed
futures = rs.pull(parties=rs_parties)
rtn = []
for party, future in zip(rs_parties, futures):
obj = future.result()
if obj is None:
raise EnvironmentError(
f"federation get None from {party} with name {name}, tag {tag}"
)
LOGGER.info(f'federation got data. name: {name}, tag: {tag}')
if isinstance(obj, RollPair):
rtn.append(obj)
rubbish.add_table(obj)
if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug(
f'federation got roll pair with count {obj.count()}, name {name}, tag {tag}'
)
elif is_split_head(obj):
num_split = obj.num_split()
LOGGER.debug(
f'federation getting split data with name {name}, tag {tag}, num split {num_split}'
)
split_objs = []
for k in range(num_split):
_split_rs = self.rsc.load(name, tag=f"{tag}.__part_{k}")
split_objs.append(_split_rs.pull([party])[0].result())
obj = split_get(split_objs)
rtn.append(obj)
else:
LOGGER.debug(
f'federation get obj with type {type(obj)} from {party} with name {name}, tag {tag}'
)
rtn.append(obj)
return rtn, rubbish
def remote(self, obj, name, tag, parties):
if isinstance(parties, Party):
parties = [parties]
self._remote_side_auth(name, parties)
if obj is None:
raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}")
rs = self.rsc.load(name=name, tag=tag)
rubbish = Rubbish(name=name, tag=tag)
rs_parties = [(party.role, party.party_id) for party in parties]
for party in parties:
if (name, tag, party) in _remote_tag_histories:
raise EnvironmentError(f"remote duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
if isinstance(obj, RollPair):
futures = rs.push(obj=obj, parties=rs_parties)
rubbish.add_table(obj)
else:
LOGGER.debug(f"federation remote obj with type {type(obj)} to {parties} with name {name}, tag {tag}")
futures = []
obj, splits = maybe_split_object(obj)
futures.extend(rs.push(obj=obj, parties=rs_parties))
for k, v in splits:
_split_rs = self.rsc.load(name, tag=f"{tag}.__part_{k}")
futures.extend(_split_rs.push(obj=v, parties=rs_parties))
def done_callback(fut):
try:
result = fut.result()
except Exception as e:
import os
import signal
import traceback
import logging
import sys
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
pid = os.getpid()
LOGGER.exception(f"remote fail, terminating process(pid={pid})")
try:
logging.shutdown()
finally:
os.kill(pid, signal.SIGTERM)
raise e
for future in futures:
future.add_done_callback(done_callback)
# warning, temporary workaround， should be remote in near released version
if not isinstance(obj, RollPair):
for obj_table in _get_remote_obj_store_table(rs_parties, rs):
rubbish.add_table(obj_table)
return rubbish
def async_get(self, name: str, tag: str,
parties: list) -> typing.Generator:
rubbish = Rubbish(name, tag)
futures = self._check_get_status_async(name, tag, parties)
for future in as_completed(futures):
party = futures[future]
obj, head, frags = future.result()
if isinstance(obj, _DTable):
rubbish.add_table(obj)
yield (party, obj)
else:
table, key = head
rubbish.add_obj(table, key)
if not is_split_head(obj):
yield (party, obj)
else:
frag_table, frag_keys = frags
rubbish.add_table(frag_table)
fragments = [frag_table.get(key) for key in frag_keys]
yield (party, split_get(fragments))
yield (None, rubbish)
def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Rubbish]:
if isinstance(parties, Party):
parties = [parties]
for party in parties:
if (name, tag, party) in _get_tag_histories:
raise EnvironmentError(f"get duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
self._get_side_auth(name=name, parties=parties)
_status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id)
LOGGER.debug(f"[GET] {self._local_party} getting {name}.{tag} from {parties}")
tasks = []
for party in parties:
_tagged_key = self.__remote__object_key(self._session_id, name, tag, party.role, party.party_id, self._role,
self._party_id)
tasks.append(check_status_and_get_value(_status_table, _tagged_key))
results = self._loop.run_until_complete(asyncio.gather(*tasks))
rtn = []
rubbish = Rubbish(name, tag)
_object_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id)
for r in results:
LOGGER.debug(f"[GET] {self._local_party} getting {r} from {parties}")
if isinstance(r, tuple):
_persistent = r[0] == StoreTypes.ROLLPAIR_LMDB
table = Standalone.get_instance().table(name=r[1], namespace=r[2], persistent=_persistent,
partition=r[3])
rtn.append(table)
rubbish.add_table(table)
else: # todo: should standalone mode split large object?
obj = _object_table.get(r)
if obj is None:
raise EnvironmentError(f"federation get None from {parties} with name {name}, tag {tag}")
rtn.append(obj)
rubbish.add_obj(_object_table, r)
rubbish.add_obj(_status_table, r)
return rtn, rubbish
def remote(self, obj, name: str, tag: str,
parties: Union[Party, list]) -> Rubbish:
if obj is None:
raise EnvironmentError(
f"federation try to remote None to {parties} with name {name}, tag {tag}"
)
if isinstance(parties, Party):
parties = [parties]
for party in parties:
if (name, tag, party) in _remote_tag_histories:
raise EnvironmentError(f"remote duplicate tag {(name, tag)}")
_remote_tag_histories.add((name, tag, party))
self._remote_side_auth(name=name, parties=parties)
rubbish = Rubbish(name, tag)
# if obj is a dtable, remote it
if isinstance(obj, _DTable):
obj.set_gc_disable()
for party in parties:
log_msg = f"src={self.local_party}, dst={party}, name={name}, tag={tag}, session_id={self._session_id}"
LOGGER.debug(f"[REMOTE] sending table: {log_msg}")
self._send(transfer_type=federation_pb2.DTABLE,
name=name,
tag=tag,
dst_party=party,
rubbish=rubbish,
table=obj)
LOGGER.debug(f"[REMOTE] table done: {log_msg}")
return rubbish
# if obj is object, put it in specified dtable, then remote the dtable
first, fragments = maybe_split_object(obj)
num_fragment = len(fragments)
if REMOTE_FRAGMENT_OBJECT_USE_D_TABLE and num_fragment > 1:
fragment_storage_table = _create_fragment_obj_table(
self._session_id)
fragment_storage_table.put_all(fragments)
for party in parties:
log_msg = f"src={self.local_party}, dst={party}, name={name}, tag={tag}, session_id={self._session_id}"
LOGGER.debug(f"[REMOTE] sending object: {log_msg}")
obj_table = _cache_remote_obj_storage_table[party]
# remote object or remote fragment header
self._send(transfer_type=federation_pb2.OBJECT,
name=name,
tag=tag,
dst_party=party,
rubbish=rubbish,
table=obj_table,
obj=first)
if not fragments:
LOGGER.debug(f"[REMOTE] object done: {log_msg}")
# remote fragments
# impl 1
if REMOTE_FRAGMENT_OBJECT_USE_D_TABLE and num_fragment > 1:
LOGGER.debug(f"[REMOTE] sending fragment table: {log_msg}")
# noinspection PyUnboundLocalVariable
self._send(transfer_type=federation_pb2.DTABLE,
name=name,
tag=f"{tag}.fragments_table",
dst_party=party,
rubbish=rubbish,
table=fragment_storage_table)
LOGGER.debug(f"[REMOTE] done fragment table: {log_msg}")
# impl 2
elif not REMOTE_FRAGMENT_OBJECT_USE_D_TABLE:
for fragment_index, fragment in fragments:
LOGGER.debug(
f"[REMOTE] sending fragment({fragment_index}/{num_fragment}): {log_msg}"
)
self._send(transfer_type=federation_pb2.OBJECT,
name=name,
tag=_fragment_tag(tag, fragment_index),
dst_party=party,
rubbish=rubbish,
table=obj_table,
obj=fragment)
LOGGER.debug(
f"[REMOTE] done fragment({fragment_index}/{num_fragment}): {log_msg}"
)
return rubbish