def get_effective_resend_option(self): """ return the resend option :return: """ if self.has_received_message() and self.has_resend_option() and \ (self.option.resend_all is not None or self.option.resend_from is not None or self.option.resend_from_time is not None): return Option(resend_from=self.last_received_offset + 1) else: return Option(resend_from_time=self.option.resend_from_time, resend_from=self.option.resend_from, resend_all=self.option.resend_all, resend_last=self.option.resend_last, resend_to=self.option.resend_to)
def gap_handler(from_, to_): """ handler when subscription detect a gap :param from_: :param to_: :return: None """ if not sub.resending: self.__request_resend(sub, Option(resend_from=from_, resend_to=to_))
def init(): """ create client with a mock connection :return: """ conn = EventMock() option = Option.get_default_option() option.auto_connect = False option.auto_disconnect = False option.api_key = api_key_test cli = Client(option, conn) return cli, conn
def get_constructor_arguments(cls, msg): """ get arguments from msg :param msg: dict :return: list """ return [ msg.get(RequestConstant.STREAM_ID, None), msg.get(RequestConstant.STREAM_PARTITION, None), msg.get(RequestConstant.SUB_ID, None), Option.deserialize_resend(msg), msg.get(RequestConstant.API_KEY, None), msg.get(RequestConstant.SESSION_TOKEN, None) ]
def test_get_original_resend_option(): def callback(_, __): """ empty callback func :param _: :param __: :return: """ print('done') sub = Subscription(stream_id, stream_partition, 'api_key', callback, Option(resend_all=True)) assert sub.get_effective_resend_option().resend_all is True
def test_resend_request(): msg = { 'sessionToken': 'my_sessionToken', 'authKey': 'authKey', 'type': 'resend', 'stream': 'id', 'partition': 0, 'sub': 'subId', 'resend_all': True, } rest = ResendRequest.deserialize(json.dumps(msg)) assert isinstance(rest, ResendRequest) assert rest.stream_id == msg['stream'] assert rest.stream_partition == msg['partition'] assert rest.sub_id == msg['sub'] assert rest.resend_option == Option(resend_all=True) assert rest.api_key == msg['authKey'] assert rest.session_token == msg['sessionToken'] msg = { 'type': 'resend', 'stream': 'id', 'partition': 0, 'sub': 'subId', 'resend_all': True, 'sessionToken': 'my_sessionToken', 'authKey': 'authKey' } serialized = ResendRequest('id', 0, 'subId', Option(resend_all=True), 'authKey', 'my_sessionToken').serialize() assert (isinstance(serialized, str)) dic = json.loads(serialized) assert dic == msg
def test_get_resend_option_after_received_data3(): def callback(_, __): """ empty callback func :param _: :param __: :return: """ print('done') sub = Subscription(stream_id, stream_partition, 'api_key', callback, Option(resend_from_time=time.time())) sub.handle_message(create_msg(10)) dic = sub.get_effective_resend_option() assert dic.resend_from == 11
def test_get_resend_option_after_received_data4(): msg = create_msg() def callback(_, __): """ empty callback func :param _: :param __: :return: """ print('done') sub = Subscription(stream_id, stream_partition, 'api_key', callback, Option(resend_last=10)) sub.handle_message(msg) dic = sub.get_effective_resend_option() assert dic.resend_last == 10
def __init__(self, stream_id=None, stream_partition=0, api_key=None, callback=None, option=None): super().__init__() if stream_id is None: raise ValueError('No stream_id given!') if api_key is None: raise ValueError('No api_key given!') if not hasattr(callback, '__call__'): raise ValueError('No callback given') self.sub_id = generate_subscription_id() self.stream_id = stream_id self.stream_partition = stream_partition self.api_key = api_key self.callback = callback if hasattr(callback, '__call__') else lambda x, y: None if isinstance(option, Option): self.option = option else: self.option = Option() self.queue = [] self.state = EventConstant.UNSUBSCRIBED self.resending = False self.last_received_offset = None if self.option.check_resend() > 1: raise ValueError( 'Multiple resend option active! Please use only one: %s' % self.option) if self.option.resend_from_time is not None: t = self.option.resend_from_time if not isinstance(t, (int, float)): raise ValueError( '"resend_from_time option" must be an int or float') def unsubscribed(): """ callback function of unsubscribed event :return: """ self.set_resending(False) self.on(EventConstant.UNSUBSCRIBED, unsubscribed) def no_resend(response=None): """ callback function of no_resend event :param response: :return: """ logger.debug('Sub %s no_resend:%s' % (self.sub_id, response)) self.set_resending(False) self.check_queue() self.on(EventConstant.NO_RESEND, no_resend) def resent(response=None): """ callback function of resent event :param response: :return: """ logger.debug('Sub %s resent: %s' % (self.sub_id, response)) self.set_resending(False) self.check_queue() self.on(EventConstant.RESENT, resent) def connected(): """ callback function of connected event :return: """ pass self.on(EventConstant.CONNECTED, connected) def disconnected(): """ callback function of disconnected event :return: """ self.set_state(EventConstant.UNSUBSCRIBED) self.set_resending(False) self.on(EventConstant.DISCONNECTED, disconnected)
class Subscription(Event): """ subscription class """ def __init__(self, stream_id=None, stream_partition=0, api_key=None, callback=None, option=None): super().__init__() if stream_id is None: raise ValueError('No stream_id given!') if api_key is None: raise ValueError('No api_key given!') if not hasattr(callback, '__call__'): raise ValueError('No callback given') self.sub_id = generate_subscription_id() self.stream_id = stream_id self.stream_partition = stream_partition self.api_key = api_key self.callback = callback if hasattr(callback, '__call__') else lambda x, y: None if isinstance(option, Option): self.option = option else: self.option = Option() self.queue = [] self.state = EventConstant.UNSUBSCRIBED self.resending = False self.last_received_offset = None if self.option.check_resend() > 1: raise ValueError( 'Multiple resend option active! Please use only one: %s' % self.option) if self.option.resend_from_time is not None: t = self.option.resend_from_time if not isinstance(t, (int, float)): raise ValueError( '"resend_from_time option" must be an int or float') def unsubscribed(): """ callback function of unsubscribed event :return: """ self.set_resending(False) self.on(EventConstant.UNSUBSCRIBED, unsubscribed) def no_resend(response=None): """ callback function of no_resend event :param response: :return: """ logger.debug('Sub %s no_resend:%s' % (self.sub_id, response)) self.set_resending(False) self.check_queue() self.on(EventConstant.NO_RESEND, no_resend) def resent(response=None): """ callback function of resent event :param response: :return: """ logger.debug('Sub %s resent: %s' % (self.sub_id, response)) self.set_resending(False) self.check_queue() self.on(EventConstant.RESENT, resent) def connected(): """ callback function of connected event :return: """ pass self.on(EventConstant.CONNECTED, connected) def disconnected(): """ callback function of disconnected event :return: """ self.set_state(EventConstant.UNSUBSCRIBED) self.set_resending(False) self.on(EventConstant.DISCONNECTED, disconnected) def check_for_gap(self, previous_offset): """ check whether some msg is missed :param previous_offset: :return: """ return previous_offset is not None \ and self.last_received_offset is not None \ and previous_offset > self.last_received_offset def handle_message(self, msg, is_resend=False): """ handle the message received from server :param msg: :param is_resend: :return: """ if msg.previous_offset is None: logger.debug( 'handle_message: prev_offset is null, gap detection is impossible! message no. %s' % msg.serialize()) if self.resending is True and is_resend is False: self.queue.append(msg) elif self.check_for_gap( msg.previous_offset) is True and self.resending is False: self.queue.append(msg) from_index = self.last_received_offset + 1 to_index = msg.previous_offset logger.debug( 'Gap detected, requesting resend for stream %s from %d to %d' % (self.stream_id, from_index, to_index)) self.emit(EventConstant.GAP, from_index, to_index) elif self.last_received_offset is not None and msg.offset <= self.last_received_offset: logger.debug( 'Sub %s already received message: %s, last_received_offset :%s. Ignoring message.' % (self.sub_id, msg.offset, self.last_received_offset)) else: self.last_received_offset = msg.offset self.callback(msg.get_parsed_content(), msg) if msg.is_bye_message(): self.emit(EventConstant.DONE) def check_queue(self): """ check whether there are data should be sent :return: """ logger.debug('Attempting to process %s queued messages for stream %s' % (len(self.queue), self.stream_id)) orig = self.queue self.queue = [] for msg in orig: self.handle_message(msg, False) def has_resend_option(self): """ check whether subscription has a resend option :return: """ return self.option.check_resend() > 0 def has_received_message(self): """ check whether subscription has received message :return: """ return self.last_received_offset is not None def get_effective_resend_option(self): """ return the resend option :return: """ if self.has_received_message() and self.has_resend_option() and \ (self.option.resend_all is not None or self.option.resend_from is not None or self.option.resend_from_time is not None): return Option(resend_from=self.last_received_offset + 1) else: return Option(resend_from_time=self.option.resend_from_time, resend_from=self.option.resend_from, resend_all=self.option.resend_all, resend_last=self.option.resend_last, resend_to=self.option.resend_to) def get_state(self): """ return the state of subscription :return: """ return self.state def set_state(self, state): """ change the state of subscription :param state: :return: """ logger.debug('Subscription: stream %s state changed %s => %s' % (self.stream_id, self.state, state)) self.state = state self.emit(state) def is_resending(self): """ check whether subscription is resending :return: """ return self.resending def set_resending(self, v): """ turn on or off the resending state :param v: :return: """ logger.debug('Subscription: Stream %s resending: %s' % (self.stream_id, v)) self.resending = v def handle_error(self, err): """ handle the error :param err: :return: """ if isinstance(err, InvalidJsonError) and not self.check_for_gap( err.stream_message.previous_offset): self.last_received_offset = err.stream_message.offset self.emit(EventConstant.ERROR, err)
def subscribe(self, stream, callback, legacy_option=None): """ subscribe to stream with given id :param stream: object or dict contains stream_id and stream_partition :param callback: callback function when subscribed :param legacy_option: backward compatibility :return: subscription """ if hasattr(stream, 'stream_id'): stream_id = stream.stream_id elif isinstance(stream, dict) and ('stream_id' in stream.keys() or 'stream' in stream.keys()): stream_id = stream.get('stream', None) or stream.get( 'stream_id', None) elif isinstance(stream, str): stream_id = stream else: raise ValueError( 'subscribe: stream_id and stream_partition should be given. Given :%s' % (type(stream))) if isinstance(legacy_option, Option): opt = copy.copy(legacy_option) opt.stream_id = stream_id else: opt = Option() sub = Subscription(stream_id, opt.stream_partition or 0, self.option.api_key, callback, opt) def gap_handler(from_, to_): """ handler when subscription detect a gap :param from_: :param to_: :return: None """ if not sub.resending: self.__request_resend(sub, Option(resend_from=from_, resend_to=to_)) sub.on(EventConstant.GAP, gap_handler) def done_handler(): """ handler when subscription is done :return: None """ logger.debug('done event for sub %s ' % sub.sub_id) self.unsubscribe(sub) sub.on(EventConstant.DONE, done_handler) self.__add_subscription(sub) if self.connection.state == EventConstant.CONNECTED: self.__resend_and_subscribe(sub) elif self.option.auto_connect is True: try: self.connect() except Exception as e: import traceback traceback.print_exc() raise ConnectionErr(e) return sub