def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Rubbish: if obj is None: raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}") if isinstance(parties, Party): parties = [parties] for party in parties: if (name, tag, party) in _remote_tag_histories: raise EnvironmentError(f"remote duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) self._remote_side_auth(name=name, parties=parties) rubbish = Rubbish(name, tag) for party in parties: _tagged_key = self.__remote__object_key(self._session_id, name, tag, self._role, self._party_id, party.role, party.party_id) _status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id) if isinstance(obj, _DTable): obj.set_gc_disable() # noinspection PyProtectedMember _status_table.put(_tagged_key, (obj._type, obj._name, obj._namespace, obj._partitions)) rubbish.add_table(obj) rubbish.add_obj(_status_table, _tagged_key) else: _table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id) _table.put(_tagged_key, obj) _status_table.put(_tagged_key, _tagged_key) rubbish.add_obj(_table, _tagged_key) rubbish.add_obj(_status_table, _tagged_key) LOGGER.debug("[REMOTE] Sent {}".format(_tagged_key)) return rubbish
def remote(self, obj, name, tag, parties): if isinstance(parties, Party): parties = [parties] self._remote_side_auth(name, parties) if obj is None: raise EnvironmentError(f"federation try to remote None to {parties} with name {name}, tag {tag}") rs = self.rsc.load(name=name, tag=tag) rubbish = Rubbish(name=name, tag=tag) rs_parties = [(party.role, party.party_id) for party in parties] for party in parties: if (name, tag, party) in _remote_tag_histories: raise EnvironmentError(f"remote duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) if isinstance(obj, RollPair): futures = rs.push(obj=obj, parties=rs_parties) rubbish.add_table(obj) else: LOGGER.debug(f"federation remote obj with type {type(obj)} to {parties} with name {name}, tag {tag}") futures = [] obj, splits = maybe_split_object(obj) futures.extend(rs.push(obj=obj, parties=rs_parties)) for k, v in splits: _split_rs = self.rsc.load(name, tag=f"{tag}.__part_{k}") futures.extend(_split_rs.push(obj=v, parties=rs_parties)) def done_callback(fut): try: result = fut.result() except Exception as e: import os import signal import traceback import logging import sys exc_info = sys.exc_info() traceback.print_exception(*exc_info) pid = os.getpid() LOGGER.exception(f"remote fail, terminating process(pid={pid})") try: logging.shutdown() finally: os.kill(pid, signal.SIGTERM) raise e for future in futures: future.add_done_callback(done_callback) # warning, temporary workaround, should be remote in near released version if not isinstance(obj, RollPair): for obj_table in _get_remote_obj_store_table(rs_parties, rs): rubbish.add_table(obj_table) return rubbish
def get(self, name, tag, parties: Union[Party, list]): rs = self.rsc.load(name=name, tag=tag) rubbish = Rubbish(name, tag) if isinstance(parties, Party): parties = [parties] rs_parties = [(party.role, party.party_id) for party in parties] for party in parties: if (name, tag, party) in _get_tag_histories: raise EnvironmentError(f"get duplicate tag {(name, tag)}") _get_tag_histories.add((name, tag, party)) # TODO:0: check if exceptions are swallowed futures = rs.pull(parties=rs_parties) rtn = [] for party, future in zip(rs_parties, futures): obj = future.result() if obj is None: raise EnvironmentError( f"federation get None from {party} with name {name}, tag {tag}" ) LOGGER.info(f'federation got data. name: {name}, tag: {tag}') if isinstance(obj, RollPair): rtn.append(obj) rubbish.add_table(obj) if LOGGER.isEnabledFor(logging.DEBUG): LOGGER.debug( f'federation got roll pair with count {obj.count()}, name {name}, tag {tag}' ) elif is_split_head(obj): num_split = obj.num_split() LOGGER.debug( f'federation getting split data with name {name}, tag {tag}, num split {num_split}' ) split_objs = [] for k in range(num_split): _split_rs = self.rsc.load(name, tag=f"{tag}.__part_{k}") split_objs.append(_split_rs.pull([party])[0].result()) obj = split_get(split_objs) rtn.append(obj) else: LOGGER.debug( f'federation get obj with type {type(obj)} from {party} with name {name}, tag {tag}' ) rtn.append(obj) return rtn, rubbish
def async_get(self, name: str, tag: str, parties: list) -> typing.Generator: rubbish = Rubbish(name, tag) futures = self._check_get_status_async(name, tag, parties) for future in as_completed(futures): party = futures[future] obj, head, frags = future.result() if isinstance(obj, _DTable): rubbish.add_table(obj) yield (party, obj) else: table, key = head rubbish.add_obj(table, key) if not is_split_head(obj): yield (party, obj) else: frag_table, frag_keys = frags rubbish.add_table(frag_table) fragments = [frag_table.get(key) for key in frag_keys] yield (party, split_get(fragments)) yield (None, rubbish)
def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Rubbish]: if isinstance(parties, Party): parties = [parties] for party in parties: if (name, tag, party) in _get_tag_histories: raise EnvironmentError(f"get duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) self._get_side_auth(name=name, parties=parties) _status_table = _get_meta_table(STATUS_TABLE_NAME, self._session_id) LOGGER.debug(f"[GET] {self._local_party} getting {name}.{tag} from {parties}") tasks = [] for party in parties: _tagged_key = self.__remote__object_key(self._session_id, name, tag, party.role, party.party_id, self._role, self._party_id) tasks.append(check_status_and_get_value(_status_table, _tagged_key)) results = self._loop.run_until_complete(asyncio.gather(*tasks)) rtn = [] rubbish = Rubbish(name, tag) _object_table = _get_meta_table(OBJECT_STORAGE_NAME, self._session_id) for r in results: LOGGER.debug(f"[GET] {self._local_party} getting {r} from {parties}") if isinstance(r, tuple): _persistent = r[0] == StoreTypes.ROLLPAIR_LMDB table = Standalone.get_instance().table(name=r[1], namespace=r[2], persistent=_persistent, partition=r[3]) rtn.append(table) rubbish.add_table(table) else: # todo: should standalone mode split large object? obj = _object_table.get(r) if obj is None: raise EnvironmentError(f"federation get None from {parties} with name {name}, tag {tag}") rtn.append(obj) rubbish.add_obj(_object_table, r) rubbish.add_obj(_status_table, r) return rtn, rubbish
def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Rubbish: if obj is None: raise EnvironmentError( f"federation try to remote None to {parties} with name {name}, tag {tag}" ) if isinstance(parties, Party): parties = [parties] for party in parties: if (name, tag, party) in _remote_tag_histories: raise EnvironmentError(f"remote duplicate tag {(name, tag)}") _remote_tag_histories.add((name, tag, party)) self._remote_side_auth(name=name, parties=parties) rubbish = Rubbish(name, tag) # if obj is a dtable, remote it if isinstance(obj, _DTable): obj.set_gc_disable() for party in parties: log_msg = f"src={self.local_party}, dst={party}, name={name}, tag={tag}, session_id={self._session_id}" LOGGER.debug(f"[REMOTE] sending table: {log_msg}") self._send(transfer_type=federation_pb2.DTABLE, name=name, tag=tag, dst_party=party, rubbish=rubbish, table=obj) LOGGER.debug(f"[REMOTE] table done: {log_msg}") return rubbish # if obj is object, put it in specified dtable, then remote the dtable first, fragments = maybe_split_object(obj) num_fragment = len(fragments) if REMOTE_FRAGMENT_OBJECT_USE_D_TABLE and num_fragment > 1: fragment_storage_table = _create_fragment_obj_table( self._session_id) fragment_storage_table.put_all(fragments) for party in parties: log_msg = f"src={self.local_party}, dst={party}, name={name}, tag={tag}, session_id={self._session_id}" LOGGER.debug(f"[REMOTE] sending object: {log_msg}") obj_table = _cache_remote_obj_storage_table[party] # remote object or remote fragment header self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=tag, dst_party=party, rubbish=rubbish, table=obj_table, obj=first) if not fragments: LOGGER.debug(f"[REMOTE] object done: {log_msg}") # remote fragments # impl 1 if REMOTE_FRAGMENT_OBJECT_USE_D_TABLE and num_fragment > 1: LOGGER.debug(f"[REMOTE] sending fragment table: {log_msg}") # noinspection PyUnboundLocalVariable self._send(transfer_type=federation_pb2.DTABLE, name=name, tag=f"{tag}.fragments_table", dst_party=party, rubbish=rubbish, table=fragment_storage_table) LOGGER.debug(f"[REMOTE] done fragment table: {log_msg}") # impl 2 elif not REMOTE_FRAGMENT_OBJECT_USE_D_TABLE: for fragment_index, fragment in fragments: LOGGER.debug( f"[REMOTE] sending fragment({fragment_index}/{num_fragment}): {log_msg}" ) self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=_fragment_tag(tag, fragment_index), dst_party=party, rubbish=rubbish, table=obj_table, obj=fragment) LOGGER.debug( f"[REMOTE] done fragment({fragment_index}/{num_fragment}): {log_msg}" ) return rubbish