def moca_loads(data: Optional[bytes]) -> Any: """Load serialized object.""" if data is None: return None elif data[:4] == b'moca': return p_loads(decompress(data[4:])) else: return p_loads(data)
def _receive(channel_info, name, tag): partitions = -1 party_id = channel_info._party_id role = channel_info._role wish_cache_key = _get_message_cache_key(name, tag, party_id, role) if wish_cache_key in message_cache: return message_cache[wish_cache_key] for method, properties, body in channel_info.consume(): LOGGER.debug( f"[rabbitmq._receive] method: {method}, properties: {properties}.") if properties.message_id != name or properties.correlation_id != tag: # todo: fix this LOGGER.warning( f"[rabbitmq._receive]: require {name}.{tag}, got {properties.message_id}.{properties.correlation_id}" ) cache_key = _get_message_cache_key(properties.message_id, properties.correlation_id, party_id, role) # object if properties.content_type == 'text/plain': message_cache[cache_key] = p_loads(body) channel_info.basic_ack(delivery_tag=method.delivery_tag) # rdd if properties.content_type == 'application/json': data = json.loads(body) data_iter = ((p_loads(bytes.fromhex(el['k'])), p_loads(bytes.fromhex(el['v']))) for el in data) sc = SparkContext.getOrCreate() partitions = properties.headers["partitions"] rdd = sc.parallelize(data_iter, partitions) if cache_key not in message_cache: message_cache[cache_key] = rdd else: message_cache[cache_key] = message_cache[cache_key].union( rdd).coalesce(partitions) # trigger action message_cache[cache_key].persist(get_storage_level()) count = message_cache[cache_key].count() LOGGER.debug(f"count: {count}") channel_info.basic_ack(delivery_tag=method.delivery_tag) # object if properties.content_type == 'text/plain': if cache_key == wish_cache_key: channel_info.cancel() return message_cache[cache_key] # rdd if properties.content_type == 'application/json': if cache_key == wish_cache_key and message_cache[cache_key].count( ) == properties.headers["total_size"]: channel_info.cancel() return message_cache[cache_key]
def _partition_receive(self, index, kvs, name, tag, party_id, role, topic_infos, mq, conf: dict): topic_pair = topic_infos[index][1] channel_info = self._get_channel(mq, topic_pair, party_id, role, conf) message_key_cache = set() count = 0 partition_size = -1 all_data = [] while True: message = channel_info.consume() properties = message.properties() # must get bytes body = message.data().decode() print(f"[pulsar._partition_receive] properties: {properties}.") if properties['message_id'] != name or properties[ 'correlation_id'] != tag: # leave this code to handle unexpected situation channel_info.basic_ack(message) print( f"[pulsar._partition_receive]: require {name}.{tag}, got {properties['message_id']}.{properties['correlation_id']}" ) continue if properties['content_type'] == 'application/json': # headers here is json bytes string header = json.loads(properties['headers']) message_key = header.get('message_key') if message_key in message_key_cache: print( f"[pulsar._partition_receive] message_key : {message_key} is duplicated" ) channel_info.basic_ack(message) continue message_key_cache.add(message_key) if header.get('partition_size') >= 0: partition_size = header.get('partition_size') data = json.loads(body) data_iter = ((p_loads(bytes.fromhex(el['k'])), p_loads(bytes.fromhex(el['v']))) for el in data) count += len(data) print(f"[pulsar._partition_receive] count: {count}") all_data.extend(data_iter) channel_info.basic_ack(message) if count == partition_size: channel_info.cancel() return all_data else: ValueError( f"[pulsar._partition_receive]properties.content_type is {properties.content_type}, but must be application/json" )
def _partition_receive(self, index, kvs, name, tag, party_id, role, party_mq_names, mq, connection_conf: dict): queue_names = party_mq_names[index][1] channel_info = self._get_channel(mq, queue_names, party_id, role, connection_conf) message_key_cache = set() count = 0 partition_size = -1 all_data = [] for method, properties, body in channel_info.consume(): print( f"[rabbitmq._partition_receive] method: {method}, properties: {properties}." ) if properties.message_id != name or properties.correlation_id != tag: # todo: fix this channel_info.basic_ack(delivery_tag=method.delivery_tag) print( f"[rabbitmq._partition_receive]: require {name}.{tag}, got {properties.message_id}.{properties.correlation_id}" ) continue if properties.content_type == 'application/json': message_key = properties.headers["message_key"] if message_key in message_key_cache: print( f"[rabbitmq._partition_receive] message_key : {message_key} is duplicated" ) channel_info.basic_ack(delivery_tag=method.delivery_tag) continue message_key_cache.add(message_key) if properties.headers["partition_size"] >= 0: partition_size = properties.headers["partition_size"] data = json.loads(body) data_iter = ((p_loads(bytes.fromhex(el['k'])), p_loads(bytes.fromhex(el['v']))) for el in data) count += len(data) print(f"[rabbitmq._partition_receive] count: {count}") all_data.extend(data_iter) channel_info.basic_ack(delivery_tag=method.delivery_tag) if count == partition_size: channel_info.cancel() return all_data else: ValueError( f"[rabbitmq._partition_receive]properties.content_type is {properties.content_type}, but must be application/json" )
def _receive_obj(self, channel_info, name, tag): party_id = channel_info._party_id role = channel_info._role wish_cache_key = self._get_message_cache_key(name, tag, party_id, role) if wish_cache_key in self._message_cache: return self._message_cache[wish_cache_key] for method, properties, body in channel_info.consume(): LOGGER.debug( f"[rabbitmq._receive_obj] method: {method}, properties: {properties}." ) if properties.message_id != name or properties.correlation_id != tag: # todo: fix this LOGGER.warning( f"[rabbitmq._receive_obj] require {name}.{tag}, got {properties.message_id}.{properties.correlation_id}" ) cache_key = self._get_message_cache_key(properties.message_id, properties.correlation_id, party_id, role) # object if properties.content_type == "text/plain": self._message_cache[cache_key] = p_loads(body) channel_info.basic_ack(delivery_tag=method.delivery_tag) if cache_key == wish_cache_key: channel_info.cancel() LOGGER.debug( f"[rabbitmq._receive_obj] cache_key: {cache_key}, obj: {self._message_cache[cache_key]}" ) return self._message_cache[cache_key] else: raise ValueError( f"[rabbitmq._receive_obj] properties.content_type is {properties.content_type}, but must be text/plain" )
def deserialize(_bytes): if _bytes: bytes_security_check(_bytes) try: return p_loads(_bytes) except: return eggroll_pickle_loads(_bytes)
def _receive_obj(self, channel_info, name, tag): party_id = channel_info._party_id role = channel_info._role wish_cache_key = self._get_message_cache_key(name, tag, party_id, role) if wish_cache_key in self._message_cache: return self._message_cache[wish_cache_key] while True: message = channel_info.consume() # return None indicates the client is closed body = message.data() properties = message.properties() LOGGER.debug(f"[pulsar._receive_obj] properties: {properties}.") if properties["message_id"] != name or properties[ "correlation_id"] != tag: LOGGER.warning( f"[pulsar._receive_obj] require {name}.{tag}, got {properties['message_id']}.{properties['correlation_id']}" ) # just ack and continue channel_info.basic_ack(message.message_id()) continue cache_key = self._get_message_cache_key( properties["message_id"], properties["correlation_id"], party_id, role) # object if properties["content_type"] == "text/plain": self._message_cache[cache_key] = p_loads(body) # TODO: handle ack failure channel_info.basic_ack(message.message_id()) if cache_key == wish_cache_key: # keep connection open for receiving object # channel_info.cancel() LOGGER.debug( f"[pulsar._receive_obj] cache_key: {cache_key}, obj: {self._message_cache[cache_key]}" ) return self._message_cache[cache_key] else: raise ValueError( f"[pulsar._receive_obj] properties.content_type is {properties.content_type}, but must be text/plain" )
def deserialize(_bytes): return p_loads(_bytes)
def get_opml(opml_path: str) -> List[str]: return p_loads(get_specific_obj(opml_path))
def deserialize(_bytes): bytes_security_check(_bytes) return p_loads(_bytes)
def _partition_receive(self, index, kvs, name, tag, party_id, role, topic_infos, mq, conf: dict): topic_pair = topic_infos[index][1] channel_info = self._get_channel(mq, topic_pair, party_id, role, conf) message_key_cache = set() count = 0 partition_size = -1 all_data = [] while True: try: message = channel_info.consume() properties = message.properties() # must get bytes body = message.data().decode() LOGGER.debug( f"[pulsar._partition_receive] properties: {properties}.") if (properties["message_id"] != name or properties["correlation_id"] != tag): # leave this code to handle unexpected situation channel_info.basic_ack(message.message_id()) LOGGER.debug( f"[pulsar._partition_receive]: require {name}.{tag}, got {properties['message_id']}.{properties['correlation_id']}" ) continue if properties["content_type"] == "application/json": # headers here is json bytes string header = json.loads(properties["headers"]) message_key = header.get("message_key") if message_key in message_key_cache: LOGGER.debug( f"[pulsar._partition_receive] message_key : {message_key} is duplicated" ) channel_info.basic_ack(message.message_id()) continue message_key_cache.add(message_key) if header.get("partition_size") >= 0: partition_size = header.get("partition_size") data = json.loads(body) data_iter = (( p_loads(bytes.fromhex(el["k"])), p_loads(bytes.fromhex(el["v"])), ) for el in data) count += len(data) LOGGER.debug( f"[pulsar._partition_receive] count: {len(data)}") LOGGER.debug( f"[pulsar._partition_receive]total count: {count}") all_data.extend(data_iter) channel_info.basic_ack(message.message_id()) if partition_size != -1: if count == partition_size: channel_info.cancel() return all_data else: raise Exception( f"[pulsar._partition_receive] want {partition_size} data in {name}.{tag} but got {count}" ) else: raise ValueError( f"[pulsar._partition_receive]properties.content_type is {properties.content_type}, but must be application/json" ) except Exception as e: LOGGER.error( f"[pulsar._partition_receive]catch exception {e}, while receiving {name}.{tag}" ) # avoid hang on consume() if count == partition_size: channel_info.cancel() return all_data else: raise e
def pickle(self, line: str = ''): """ Pickles a variable and copies it to the clipboard or un-pickles clipboard contents and prints or stores it. `%pickle` unpickle clipboard and print `%pickle v` pickle variable `v` and store in clipboard `%pickle _` pickle last line's output and store in clipboard `%pickle -o my_var` unpickle clipboard contents and store in `my_var`""" ip = self.shell args = magic_args.parse_argstring(self.pickle, line) if bool(args.output) and bool(args.var): msg = ( 'Incorrect usage, you can either pickle a variable, or unpickle, but not both at the same time.\n' f'\n`%pickle {args.var}` to pickle the contents of `{args.var}` and send them to your clipboard' f'\n`%pickle -o {args.output[0]}` to unpickle clipboard contents and send them to `{args.output[0]}`' f'\n`%pickle` to unpickle your clipboard contents and print') ip.write_err(msg) return None if not line or args.output: # user wants to unpickle from clipboard content: str = pypaste() format_error = not content.startswith( 'b') and content[1] != content[-1] # b'...' or b"..." if format_error: # clipboard doesn't have a valid pickle string sys.stderr.write( r'''Your clipboard doesn't have a bytes-like string (ie. b'\x80\x03N.' or b"\x80\x03N.")''') return None if not content: # clipboard is empty sys.stderr.write(r'Your clipboard is empty.') return None try: unpickled = p_loads((literal_eval(content))) except (KeyError, UnpicklingError, PickleError): sys.stderr.write( r'Your clipboard contents could not be unpickled because the data is not valid.' ) else: if args.output: # user wants to unpickle into a variable ip.user_ns[args.output[0]] = unpickled else: # user wants to unpickle and print sys.stdout.write(str(unpickled)) else: # user wants to pickle a var try: pickled_data = str(p_dumps(ip.user_ns.get(args.var))) except RuntimeError: sys.stderr.write( "Your data could not be pickled because it may be highly recursive.\n" "For more information on what can be (un)pickled checkout " "https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled" ) except PicklingError: sys.stderr.write( "The object you are trying to pickle is unpickable.\n" "For more information on what can be (un)pickled checkout " "https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled" ) else: pycopy(pickled_data)