def get_effective_resend_option(self):
"""
return the resend option
:return:
"""
if self.has_received_message() and self.has_resend_option() and \
(self.option.resend_all is not None or
self.option.resend_from is not None or
self.option.resend_from_time is not None):
return Option(resend_from=self.last_received_offset + 1)
else:
return Option(resend_from_time=self.option.resend_from_time,
resend_from=self.option.resend_from,
resend_all=self.option.resend_all,
resend_last=self.option.resend_last,
resend_to=self.option.resend_to)
def gap_handler(from_, to_):
"""
handler when subscription detect a gap
:param from_:
:param to_:
:return: None
"""
if not sub.resending:
self.__request_resend(sub,
Option(resend_from=from_, resend_to=to_))
def init():
"""
create client with a mock connection
:return:
"""
conn = EventMock()
option = Option.get_default_option()
option.auto_connect = False
option.auto_disconnect = False
option.api_key = api_key_test
cli = Client(option, conn)
return cli, conn
def get_constructor_arguments(cls, msg):
"""
get arguments from msg
:param msg: dict
:return: list
"""
return [
msg.get(RequestConstant.STREAM_ID, None),
msg.get(RequestConstant.STREAM_PARTITION, None),
msg.get(RequestConstant.SUB_ID, None),
Option.deserialize_resend(msg),
msg.get(RequestConstant.API_KEY, None),
msg.get(RequestConstant.SESSION_TOKEN, None)
]
def test_get_original_resend_option():
def callback(_, __):
"""
empty callback func
:param _:
:param __:
:return:
"""
print('done')
sub = Subscription(stream_id, stream_partition, 'api_key', callback,
Option(resend_all=True))
assert sub.get_effective_resend_option().resend_all is True
def test_resend_request():
msg = {
'sessionToken': 'my_sessionToken',
'authKey': 'authKey',
'type': 'resend',
'stream': 'id',
'partition': 0,
'sub': 'subId',
'resend_all': True,
}
rest = ResendRequest.deserialize(json.dumps(msg))
assert isinstance(rest, ResendRequest)
assert rest.stream_id == msg['stream']
assert rest.stream_partition == msg['partition']
assert rest.sub_id == msg['sub']
assert rest.resend_option == Option(resend_all=True)
assert rest.api_key == msg['authKey']
assert rest.session_token == msg['sessionToken']
msg = {
'type': 'resend',
'stream': 'id',
'partition': 0,
'sub': 'subId',
'resend_all': True,
'sessionToken': 'my_sessionToken',
'authKey': 'authKey'
}
serialized = ResendRequest('id', 0, 'subId', Option(resend_all=True),
'authKey', 'my_sessionToken').serialize()
assert (isinstance(serialized, str))
dic = json.loads(serialized)
assert dic == msg
def test_get_resend_option_after_received_data3():
def callback(_, __):
"""
empty callback func
:param _:
:param __:
:return:
"""
print('done')
sub = Subscription(stream_id, stream_partition, 'api_key', callback,
Option(resend_from_time=time.time()))
sub.handle_message(create_msg(10))
dic = sub.get_effective_resend_option()
assert dic.resend_from == 11
def test_get_resend_option_after_received_data4():
msg = create_msg()
def callback(_, __):
"""
empty callback func
:param _:
:param __:
:return:
"""
print('done')
sub = Subscription(stream_id, stream_partition, 'api_key', callback,
Option(resend_last=10))
sub.handle_message(msg)
dic = sub.get_effective_resend_option()
assert dic.resend_last == 10
def __init__(self,
stream_id=None,
stream_partition=0,
api_key=None,
callback=None,
option=None):
super().__init__()
if stream_id is None:
raise ValueError('No stream_id given!')
if api_key is None:
raise ValueError('No api_key given!')
if not hasattr(callback, '__call__'):
raise ValueError('No callback given')
self.sub_id = generate_subscription_id()
self.stream_id = stream_id
self.stream_partition = stream_partition
self.api_key = api_key
self.callback = callback if hasattr(callback,
'__call__') else lambda x, y: None
if isinstance(option, Option):
self.option = option
else:
self.option = Option()
self.queue = []
self.state = EventConstant.UNSUBSCRIBED
self.resending = False
self.last_received_offset = None
if self.option.check_resend() > 1:
raise ValueError(
'Multiple resend option active! Please use only one: %s' %
self.option)
if self.option.resend_from_time is not None:
t = self.option.resend_from_time
if not isinstance(t, (int, float)):
raise ValueError(
'"resend_from_time option" must be an int or float')
def unsubscribed():
"""
callback function of unsubscribed event
:return:
"""
self.set_resending(False)
self.on(EventConstant.UNSUBSCRIBED, unsubscribed)
def no_resend(response=None):
"""
callback function of no_resend event
:param response:
:return:
"""
logger.debug('Sub %s no_resend:%s' % (self.sub_id, response))
self.set_resending(False)
self.check_queue()
self.on(EventConstant.NO_RESEND, no_resend)
def resent(response=None):
"""
callback function of resent event
:param response:
:return:
"""
logger.debug('Sub %s resent: %s' % (self.sub_id, response))
self.set_resending(False)
self.check_queue()
self.on(EventConstant.RESENT, resent)
def connected():
"""
callback function of connected event
:return:
"""
pass
self.on(EventConstant.CONNECTED, connected)
def disconnected():
"""
callback function of disconnected event
:return:
"""
self.set_state(EventConstant.UNSUBSCRIBED)
self.set_resending(False)
self.on(EventConstant.DISCONNECTED, disconnected)
class Subscription(Event):
"""
subscription class
"""
def __init__(self,
stream_id=None,
stream_partition=0,
api_key=None,
callback=None,
option=None):
super().__init__()
if stream_id is None:
raise ValueError('No stream_id given!')
if api_key is None:
raise ValueError('No api_key given!')
if not hasattr(callback, '__call__'):
raise ValueError('No callback given')
self.sub_id = generate_subscription_id()
self.stream_id = stream_id
self.stream_partition = stream_partition
self.api_key = api_key
self.callback = callback if hasattr(callback,
'__call__') else lambda x, y: None
if isinstance(option, Option):
self.option = option
else:
self.option = Option()
self.queue = []
self.state = EventConstant.UNSUBSCRIBED
self.resending = False
self.last_received_offset = None
if self.option.check_resend() > 1:
raise ValueError(
'Multiple resend option active! Please use only one: %s' %
self.option)
if self.option.resend_from_time is not None:
t = self.option.resend_from_time
if not isinstance(t, (int, float)):
raise ValueError(
'"resend_from_time option" must be an int or float')
def unsubscribed():
"""
callback function of unsubscribed event
:return:
"""
self.set_resending(False)
self.on(EventConstant.UNSUBSCRIBED, unsubscribed)
def no_resend(response=None):
"""
callback function of no_resend event
:param response:
:return:
"""
logger.debug('Sub %s no_resend:%s' % (self.sub_id, response))
self.set_resending(False)
self.check_queue()
self.on(EventConstant.NO_RESEND, no_resend)
def resent(response=None):
"""
callback function of resent event
:param response:
:return:
"""
logger.debug('Sub %s resent: %s' % (self.sub_id, response))
self.set_resending(False)
self.check_queue()
self.on(EventConstant.RESENT, resent)
def connected():
"""
callback function of connected event
:return:
"""
pass
self.on(EventConstant.CONNECTED, connected)
def disconnected():
"""
callback function of disconnected event
:return:
"""
self.set_state(EventConstant.UNSUBSCRIBED)
self.set_resending(False)
self.on(EventConstant.DISCONNECTED, disconnected)
def check_for_gap(self, previous_offset):
"""
check whether some msg is missed
:param previous_offset:
:return:
"""
return previous_offset is not None \
and self.last_received_offset is not None \
and previous_offset > self.last_received_offset
def handle_message(self, msg, is_resend=False):
"""
handle the message received from server
:param msg:
:param is_resend:
:return:
"""
if msg.previous_offset is None:
logger.debug(
'handle_message: prev_offset is null, gap detection is impossible! message no. %s'
% msg.serialize())
if self.resending is True and is_resend is False:
self.queue.append(msg)
elif self.check_for_gap(
msg.previous_offset) is True and self.resending is False:
self.queue.append(msg)
from_index = self.last_received_offset + 1
to_index = msg.previous_offset
logger.debug(
'Gap detected, requesting resend for stream %s from %d to %d' %
(self.stream_id, from_index, to_index))
self.emit(EventConstant.GAP, from_index, to_index)
elif self.last_received_offset is not None and msg.offset <= self.last_received_offset:
logger.debug(
'Sub %s already received message: %s, last_received_offset :%s. Ignoring message.'
% (self.sub_id, msg.offset, self.last_received_offset))
else:
self.last_received_offset = msg.offset
self.callback(msg.get_parsed_content(), msg)
if msg.is_bye_message():
self.emit(EventConstant.DONE)
def check_queue(self):
"""
check whether there are data should be sent
:return:
"""
logger.debug('Attempting to process %s queued messages for stream %s' %
(len(self.queue), self.stream_id))
orig = self.queue
self.queue = []
for msg in orig:
self.handle_message(msg, False)
def has_resend_option(self):
"""
check whether subscription has a resend option
:return:
"""
return self.option.check_resend() > 0
def has_received_message(self):
"""
check whether subscription has received message
:return:
"""
return self.last_received_offset is not None
def get_effective_resend_option(self):
"""
return the resend option
:return:
"""
if self.has_received_message() and self.has_resend_option() and \
(self.option.resend_all is not None or
self.option.resend_from is not None or
self.option.resend_from_time is not None):
return Option(resend_from=self.last_received_offset + 1)
else:
return Option(resend_from_time=self.option.resend_from_time,
resend_from=self.option.resend_from,
resend_all=self.option.resend_all,
resend_last=self.option.resend_last,
resend_to=self.option.resend_to)
def get_state(self):
"""
return the state of subscription
:return:
"""
return self.state
def set_state(self, state):
"""
change the state of subscription
:param state:
:return:
"""
logger.debug('Subscription: stream %s state changed %s => %s' %
(self.stream_id, self.state, state))
self.state = state
self.emit(state)
def is_resending(self):
"""
check whether subscription is resending
:return:
"""
return self.resending
def set_resending(self, v):
"""
turn on or off the resending state
:param v:
:return:
"""
logger.debug('Subscription: Stream %s resending: %s' %
(self.stream_id, v))
self.resending = v
def handle_error(self, err):
"""
handle the error
:param err:
:return:
"""
if isinstance(err, InvalidJsonError) and not self.check_for_gap(
err.stream_message.previous_offset):
self.last_received_offset = err.stream_message.offset
self.emit(EventConstant.ERROR, err)
def subscribe(self, stream, callback, legacy_option=None):
"""
subscribe to stream with given id
:param stream: object or dict contains stream_id and stream_partition
:param callback: callback function when subscribed
:param legacy_option: backward compatibility
:return: subscription
"""
if hasattr(stream, 'stream_id'):
stream_id = stream.stream_id
elif isinstance(stream, dict) and ('stream_id' in stream.keys()
or 'stream' in stream.keys()):
stream_id = stream.get('stream', None) or stream.get(
'stream_id', None)
elif isinstance(stream, str):
stream_id = stream
else:
raise ValueError(
'subscribe: stream_id and stream_partition should be given. Given :%s'
% (type(stream)))
if isinstance(legacy_option, Option):
opt = copy.copy(legacy_option)
opt.stream_id = stream_id
else:
opt = Option()
sub = Subscription(stream_id, opt.stream_partition or 0,
self.option.api_key, callback, opt)
def gap_handler(from_, to_):
"""
handler when subscription detect a gap
:param from_:
:param to_:
:return: None
"""
if not sub.resending:
self.__request_resend(sub,
Option(resend_from=from_, resend_to=to_))
sub.on(EventConstant.GAP, gap_handler)
def done_handler():
"""
handler when subscription is done
:return: None
"""
logger.debug('done event for sub %s ' % sub.sub_id)
self.unsubscribe(sub)
sub.on(EventConstant.DONE, done_handler)
self.__add_subscription(sub)
if self.connection.state == EventConstant.CONNECTED:
self.__resend_and_subscribe(sub)
elif self.option.auto_connect is True:
try:
self.connect()
except Exception as e:
import traceback
traceback.print_exc()
raise ConnectionErr(e)
return sub