def _run(self, N):
channel = make_channel_name('two_identical_subscribers')
with make_client(endpoint=endpoint, appkey=appkey) as pub:
with make_client(endpoint=endpoint, appkey=appkey,
protocol='cbor') as sub1:
with make_client(endpoint=endpoint, appkey=appkey) as sub2:
origin = sync_publish(pub, channel, u'prime')
so1 = sync_subscribe(sub1, channel, {u'position': origin})
so2 = sync_subscribe(sub2, channel, {u'position': origin})
for i in range(N):
pub.publish(channel, i)
msgs1 = []
msgs2 = []
origin = time.time()
while time.time() < origin + 5:
msgs1 = so1.extract_received_messages()
msgs2 = so2.extract_received_messages()
if len(msgs1) == N + 1 and len(msgs2) == N + 1:
break
time.sleep(0.1)
self.assertEqual(msgs1, msgs2)
def generic_test_two_clients_with_message_list(self, message_list):
with make_client(endpoint=endpoint,
appkey=appkey,
reconnect_interval=0) as subscriber:
with make_client(endpoint=endpoint, appkey=appkey) as publisher:
co1 = ClientObserver()
subscriber.observer = co1
co2 = ClientObserver()
publisher.observer = co2
so = sync_subscribe(subscriber, channel)
for msg in message_list:
publisher.publish(channel, msg)
sync_publish(publisher, channel, 'finalizer')
while 'finalizer' !=\
so.last_received_channel_data['messages'][-1]:
print(so.last_received_channel_data)
last_data = so.wait_for_channel_data()
if last_data['messages'][-1] == 'finalizer':
break
got_messages = so.extract_received_messages()
self.assertEqual(got_messages, message_list + ['finalizer'])
def test_two_clients_with_deduplication(self):
with make_client(endpoint=endpoint, appkey=appkey) as pub:
with make_client(endpoint=endpoint, appkey=appkey) as sub:
so = sync_subscribe(sub,
channel,
args={'only': 'value_changes'})
pub.publish(channel, "first")
so.wait_for_channel_data()
for _ in range(10):
pub.publish(channel, "second")
so.wait_for_channel_data()
pub.publish(channel, "third")
so.wait_for_channel_data()
got_messages = so.extract_received_messages()
self.assertEqual(got_messages, ['first', 'second', 'third'])
def main():
with make_client(endpoint=endpoint,
appkey=application_key.appkey) as client:
print('Connected to Satori RTM!')
class SubscriptionObserver(object):
def on_subscription_data(self, data):
for message in data['messages']:
if (message['data']['network'] == 'DASH'):
try:
producer.send(topic2, json.dumps(message))
producer.flush()
except:
producer.send(topic, json.dumps(message))
producer.flush()
elif (message['data']['network'] == 'LTC'
or message['data']['network'] == 'DOGE'):
producer.send(topic2, json.dumps(message))
producer.flush()
else:
producer.send(topic, json.dumps(message))
producer.flush()
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
def main():
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected to Satori RTM!')
class SubscriptionObserver(object):
# Called when the subscription is established.
def on_enter_subscribed(self):
print "Subscribed to the channel '{}' with send_fraction {}".format(
channel, send_fraction)
def on_enter_failed(self, reason):
print 'Subscription failed, reason:', reason
sys.exit(1)
def on_subscription_data(self, data):
count = 0
for i, message in enumerate(data['messages']):
# send message to kafka
if i % (1 / send_fraction) == 0:
producer.send('weather', key='weather', value=message)
count += 1
print 'sending {} msgs in batch'.format(count)
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
try:
while True:
time.sleep(1)
pass
except KeyboardInterrupt:
pass
def test_channel_search(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
event = threading.Event()
mailbox = []
def callback(ack):
mailbox.append(ack)
if ack['action'] != 'rtm/search/data':
event.set()
sync_publish(client, channel, 'ping')
client.search('', callback=callback)
event.wait(10)
self.assertTrue(len(mailbox) > 0)
last_pdu = mailbox[-1]
self.assertEqual(last_pdu['action'], 'rtm/search/ok')
for non_last_pdu in mailbox[:-1]:
self.assertEqual(non_last_pdu['action'], 'rtm/search/data')
result = []
for pdu in mailbox:
result += pdu['body']['channels']
self.assertTrue(channel in result)
def test_wrong_subscribe_ack(self):
with make_client(endpoint=endpoint,
appkey=appkey,
reconnect_interval=10) as client:
client.observer = ClientObserver()
old_received_message =\
client._internal.connection.on_incoming_json
client._internal.connection.on_incoming_json =\
lambda *args: None
so = SubscriptionObserver()
client.subscribe('test_wrong_subscribe_ack',
SubscriptionMode.ADVANCED, so)
client._queue.join()
old_received_message({
u"action": u"rtm/publish/ok",
u"body": {},
u"id": 0
})
client._queue.join()
expected_log = ['on_leave_connected', 'on_enter_awaiting']
self.assertEqual(client.observer.log[:2], expected_log)
def test_write_read(self):
with make_client(endpoint, appkey) as client:
k = make_channel_name('kv')
v = make_channel_name('message')
mailbox = []
event = Event()
def callback(ack):
mailbox.append(ack)
event.set()
client.write(k, v, callback=callback)
event.wait(10)
event.clear()
client.read(k, callback=callback)
event.wait(10)
assert len(mailbox) == 2
write_ack = mailbox[0]
read_ack = mailbox[1]
assert write_ack['action'] == 'rtm/write/ok'
assert read_ack['action'] == 'rtm/read/ok'
assert read_ack['body']['message'] == v
def satori_client():
# connect to transportation data stream at satori
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected to channel: {}'.format(topic))
mailbox = []
got_message_event = threading.Event()
class SubscriptionObserver(object):
def on_subscription_data(self, data):
for message in data['messages']:
mailbox.append(message)
got_message_event.set()
# instantiate class
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
if not got_message_event.wait(20):
print('Message timeout exceeded.')
sys.exit(1)
# return json in mailbox object
while mailbox:
send_json(json.dumps(mailbox[-1], sort_keys=True).encode('utf-8'))
def test_concurrent_publishes_in_different_clients(self):
mailbox2 = []
self.publish_errors = []
def work():
with make_client(endpoint=endpoint, appkey=appkey) as publisher:
for i in six.moves.range(100):
try:
sync_publish(publisher, channel, message)
except Exception as e:
self.publish_errors.append(e)
with make_client(endpoint=endpoint, appkey=appkey) as subscriber:
so = sync_subscribe(subscriber, channel)
so.on_subscription_data = lambda data:\
mailbox2.extend(data['messages'])
threads = []
for i in range(10):
threads.append(threading.Thread(target=work))
for t in threads:
t.start()
for t in threads:
t.join()
time.sleep(3)
self.assertEqual([], self.publish_errors)
# assertEqual seems to choke on 1000 of 'hello's
self.assertEqual(len(mailbox2), 1000)
self.assertTrue(all((m == 'hello' for m in mailbox2)))
def test_concurrent_publishes_in_one_client(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
mailbox = []
self.publish_errors = []
so = sync_subscribe(client, channel)
so.on_subscription_data = lambda data:\
mailbox.extend(data['messages'])
def work():
for i in six.moves.range(100):
try:
sync_publish(client, channel, message)
except Exception as e:
self.publish_errors.append(e)
threads = []
for i in range(10):
threads.append(threading.Thread(target=work))
for t in threads:
t.start()
for t in threads:
t.join()
print('Waiting for the rest of messages')
time.sleep(3)
print("We've hopefully waited long enough")
self.assertEqual([], self.publish_errors)
# assertEqual seems to choke on 1000 of 'hello's
self.assertEqual(len(mailbox), 1000)
self.assertTrue(all((m == 'hello' for m in mailbox)))
def ask_price_ether(intent, session):
with make_client(endpoint=endpoint, appkey=appkey) as client:
dollars, cents = 0, 0
tick = []
event = threading.Event()
def read_callback(reply):
tick.append(reply)
event.set()
client.read(channel, callback=read_callback)
if not event.wait(2):
print('Read request timed out')
else:
price = tick[0]['body']['message']['price']
exchange = tick[0]['body']['message']['exchange']
dollars = int(price)
cents = int((price - dollars)*100) #room for improvement, truncates for now instead of rounding fractional pennies
session_attributes = {}
reprompt_text = None
speech_output = "Last Price was " + str(dollars) + \
" dollars and " + str(cents) + " cents, reported by the " + exchange + " exchange."
should_end_session = False
return build_response(session_attributes, build_speechlet_response(
intent['name'], speech_output, reprompt_text, should_end_session))
def main():
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected!')
mailbox = []
got_message_event = threading.Event()
class SubscriptionObserver(object):
def on_subscription_data(self, data):
for message in data['messages']:
mailbox.append(message)
got_message_event.set()
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
if not got_message_event.wait(10):
print("Timeout while waiting for a message")
sys.exit(1)
while True:
for message in mailbox:
msg = json.dumps(message, ensure_ascii=False)
producer.send(topic, msg.encode())
# do not send the messages to fast for development
time.sleep(0.1)
def main():
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected!')
mailbox = []
got_message_event = threading.Event()
class SubscriptionObserver(object):
def on_subscription_data(self, data):
for message in data['messages']:
mailbox.append(message)
#response = client1.put_record(DeliveryStreamName='traffic_stream',Record={'Data': json.dumps(message) + '\n'})
got_message_event.set()
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
if not got_message_event.wait(30):
print("Timeout while waiting for a message")
sys.exit(1)
for message in mailbox:
response = client1.put_record(
DeliveryStreamName='traffic_stream',
Record={'Data': json.dumps(message) + '\n'})
print(message)
def main():
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected to Satori RTM!')
class SubscriptionObserver(object):
def on_subscription_data(self, data):
ADDRESS = ("amqps://"
"send"
":"
"RJQLhw9Uu8qHL0KZQkb0dd+4didTvGqIzJZljtP5m/s="
"@"
"structstreaming.servicebus.windows.net"
"/"
"twitter")
sender = Sender()
azClient = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \
.publish(sender) \
.run_daemon()
for message in data['messages']:
sender.send(EventData(str(message)))
print("Got message:", message)
subscription_observer = SubscriptionObserver()
client.subscribe(channel, SubscriptionMode.SIMPLE,
subscription_observer)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
def test_exception_in_subscription_callback(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
exit = threading.Event()
class CrashyObserver(SubscriptionObserver):
def on_deleted(this):
SubscriptionObserver.on_deleted(this)
exit.set()
raise ValueError('Error in on_deleted')
def on_enter_subscribing(this):
SubscriptionObserver.on_enter_subscribing(this)
raise ValueError('Error in on_enter_subscribing')
so = CrashyObserver()
client.subscribe(channel, SubscriptionMode.ADVANCED, so)
so.wait_subscribed()
client.unsubscribe(channel)
client.subscribe(channel, SubscriptionMode.ADVANCED,
CrashyObserver())
client.unsubscribe(channel)
client.subscribe(channel,
SubscriptionMode.ADVANCED,
SubscriptionObserver(),
args={'position': '123'})
so.wait_deleted()
client.unsubscribe(channel)
if not exit.wait(20):
raise RuntimeError('Timeout')
def work():
with make_client(endpoint=endpoint, appkey=appkey) as publisher:
for i in six.moves.range(100):
try:
sync_publish(publisher, channel, message)
except Exception as e:
self.publish_errors.append(e)
def test_missing_subscription_observer_callbacks_are_fine(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
client.subscribe(
make_channel_name('missing_subscription_callbacks'),
SubscriptionMode.ADVANCED, object())
time.sleep(3)
def test_client_state_callbacks(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
global step
step = 1
class TestClientObserver(ClientObserver):
def on_enter_stopped(this):
ClientObserver.on_enter_stopped(this)
global step
if step == 1:
step = 2
client.start()
elif step == 3:
exit.set()
def on_enter_connected(this):
ClientObserver.on_enter_connected(this)
global step
if step == 2:
step = 3
client.stop()
co = TestClientObserver()
client.observer = co
exit = threading.Event()
stop_timer = threading.Timer(0.01, lambda: client.stop())
stop_timer.daemon = True
stop_timer.name = 'StopTimer'
stop_timer.start()
if not exit.wait(20):
raise RuntimeError('Timeout')
def test_write_delete_read(self):
with make_client(endpoint, appkey) as client:
k = make_channel_name('delete_read')
v = make_channel_name('value')
mailbox = []
event = Event()
def callback(ack):
mailbox.append(ack)
event.set()
client.write(k, v, callback=callback)
event.wait(10)
event.clear()
client.delete(k, callback=callback)
event.wait(10)
event.clear()
client.read(k, callback=callback)
event.wait(10)
assert len(mailbox) == 3
assert mailbox[0]['action'] == 'rtm/write/ok'
assert mailbox[1]['action'] == 'rtm/delete/ok'
assert mailbox[2]['action'] == 'rtm/read/ok'
assert mailbox[2]['body']['message'] is None
def test_publish_to_restricted_channel(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
auth_event = threading.Event()
auth_delegate = auth.RoleSecretAuthDelegate(role, secret)
mailbox = []
def auth_callback(auth_result):
if type(auth_result) == auth.Done:
mailbox.append('Auth success')
auth_event.set()
else:
mailbox.append('Auth failure: {0}'.format(auth_result))
auth_event.set()
client.authenticate(auth_delegate, auth_callback)
if not auth_event.wait(60):
raise RuntimeError('Auth never finished')
if not mailbox == ['Auth success']:
raise RuntimeError(mailbox)
sync_publish(client, channel, 'ohai')
def test_quickly_change_observer_twice(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
channel = make_channel_name('change_observer_twice')
so2 = SubscriptionObserver()
so1 = sync_subscribe(client, channel)
client.unsubscribe(channel)
client.subscribe(channel, SubscriptionMode.ADVANCED, so2)
client.unsubscribe(channel)
client.subscribe(channel, SubscriptionMode.ADVANCED, None)
client.unsubscribe(channel)
so3 = sync_subscribe(client, channel)
expected_so1_log = [
'on_leave_unsubscribed', 'on_enter_subscribing',
'on_leave_subscribing', 'on_enter_subscribed',
'on_leave_subscribed', 'on_enter_unsubscribing',
'on_leave_unsubscribing', 'on_enter_unsubscribed', 'on_deleted'
]
expected_so2_log = ['on_deleted']
expected_so3_log = [
'on_created', 'on_leave_unsubscribed', 'on_enter_subscribing',
'on_leave_subscribing', 'on_enter_subscribed'
]
self.assertEqual(so1.log, expected_so1_log)
self.assertEqual(so2.log, expected_so2_log)
self.assertEqual(so3.log, expected_so3_log)
def test_double_unsubscribe_error(self):
with make_client(
endpoint=endpoint, appkey=appkey) as client:
client.unsubscribe(channel)
so = sync_subscribe(client, channel)
client.unsubscribe(channel)
so.wait_deleted()
expected_log = [
'on_leave_unsubscribed',
'on_enter_subscribing',
'on_leave_subscribing',
'on_enter_subscribed',
'on_leave_subscribed',
'on_enter_unsubscribing',
'on_leave_unsubscribing',
'on_enter_unsubscribed',
'on_deleted'
]
self.assertEqual(so.log, expected_log)
self.assertTrue(
channel not in client._internal.subscriptions)
client.unsubscribe(channel)
def main():
with make_client(
endpoint=endpoint, appkey=appkey) as client:
print('Connected!')
mailbox = []
got_message_event = threading.Event()
class SubscriptionObserver(object):
def on_subscription_data(self, data):
for message in data['messages']:
mailbox.append(message)
got_message_event.set()
subscription_observer = SubscriptionObserver()
client.subscribe(
channel,
SubscriptionMode.SIMPLE,
subscription_observer)
if not got_message_event.wait(10):
print("Timeout while waiting for a message")
sys.exit(1)
for message in mailbox:
print('Got message "{0}"'.format(message))
def test_unsubscribe_nack_error(self):
with make_client(
endpoint=endpoint, appkey=appkey) as client:
client.unsubscribe(channel)
so = sync_subscribe(client, channel)
old_received_message =\
client._internal.connection.on_incoming_text_frame
client._internal.connection.on_incoming_text_frame =\
lambda *args: None
client.unsubscribe(channel)
client._queue.join()
old_received_message(
b'{"action":"rtm/unsubscribe/error","body":{},"id":1}')
client._queue.join()
expected_log = [
'on_leave_unsubscribed',
'on_enter_subscribing',
'on_leave_subscribing',
'on_enter_subscribed',
'on_leave_subscribed',
'on_enter_unsubscribing',
'on_leave_unsubscribing',
'on_enter_subscribed']
self.assertEqual(so.log, expected_log)
def test_wrong_unsubscribe_ack(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
client.observer = ClientObserver()
sync_subscribe(client, channel)
old_received_message =\
client._internal.connection.on_incoming_text_frame
client._internal.connection.on_incoming_text_frame =\
lambda *args: None
client.unsubscribe(channel)
client._queue.join()
old_received_message(
b'{"action":"rtm/publish/ok","body":{},"id":1}')
client._queue.join()
client.observer.wait_connected()
expected_log = [
'on_leave_connected',
'on_enter_awaiting',
'on_leave_awaiting',
'on_enter_connecting',
'on_leave_connecting',
'on_enter_connected',
]
self.assertEqual(client.observer.log, expected_log)
def bad_connect():
with make_client(endpoint='ws://qwerty:9999',
appkey=appkey,
reconnect_interval=0,
fail_count_threshold=2):
pass
def main():
import logging
logging.basicConfig(level=logging.WARNING)
class SubscriptionObserver(object):
def on_enter_subscribed(self):
print('Subscribed to: animals')
def on_subscription_data(self, pdu):
for animal in pdu['messages']:
print('Got animal {0}: {1}'.format(animal['who'], animal))
def on_enter_failed(self, reason):
print('Subscription failed:', reason, file=sys.stderr)
with make_client(endpoint=endpoint, appkey=appkey) as client:
print('Connected to Satori RTM!')
observer = SubscriptionObserver()
client.subscribe(u'animals', SubscriptionMode.SIMPLE, observer)
print('Press CTRL-C to exit', file=sys.stderr)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
def test_channel_error_during_unsubscribing(self):
with make_client(endpoint=endpoint, appkey=appkey) as client:
so = sync_subscribe(client, channel)
client._internal.subscriptions[channel]\
.on_unsubscribe_ok = lambda *args: None
client.unsubscribe(channel)
client._queue.join()
emulate_channel_error(client, channel)
so.wait_not_subscribed()
sync_publish(client, channel, u'should-be-missed')
so.wait_not_subscribed()
expected_log = [
'on_leave_unsubscribed',
'on_enter_subscribing',
'on_leave_subscribing',
'on_enter_subscribed',
'on_leave_subscribed',
'on_enter_unsubscribing',
'on_leave_unsubscribing',
'on_enter_unsubscribed',
'on_deleted'
]
self.assertEqual(so.log, expected_log)
def test_two_clients_with_best_effort_delivery(self):
with make_client(endpoint=endpoint,
appkey=appkey,
reconnect_interval=0) as subscriber:
with make_client(endpoint=endpoint, appkey=appkey) as publisher:
co1 = ClientObserver()
subscriber.observer = co1
co2 = ClientObserver()
publisher.observer = co2
try:
so = sync_subscribe(subscriber,
channel,
mode=SubscriptionMode.RELIABLE)
sync_publish(publisher, channel, 'first-message')
so.wait_for_channel_data('First receive timeout')
emulate_websocket_disconnect(subscriber)
so.wait_not_subscribed()
# send a message while subscriber is disconnected
sync_publish(publisher, channel, 'second-message')
so.wait_subscribed('Second subscribe timeout')
so.wait_for_channel_data('Second receive timeout')
# send a message after subscribed reconnected
publisher.publish(channel, 'third-message')
so.wait_for_channel_data('Third receive timeout')
expected_messages =\
['first-message', 'second-message', 'third-message']
got_messages = []
for log_entry in so.log:
if log_entry[0] == 'data':
got_messages += log_entry[1]['messages']
self.assertEqual(got_messages, expected_messages)
except Exception:
print('Subscriber log: {0}'.format(co1.log))
print('Publisher log: {0}'.format(co2.log))
print('Subscription log: {0}'.format(so.log))
raise
