def _run(self, N): channel = make_channel_name('two_identical_subscribers') with make_client(endpoint=endpoint, appkey=appkey) as pub: with make_client(endpoint=endpoint, appkey=appkey, protocol='cbor') as sub1: with make_client(endpoint=endpoint, appkey=appkey) as sub2: origin = sync_publish(pub, channel, u'prime') so1 = sync_subscribe(sub1, channel, {u'position': origin}) so2 = sync_subscribe(sub2, channel, {u'position': origin}) for i in range(N): pub.publish(channel, i) msgs1 = [] msgs2 = [] origin = time.time() while time.time() < origin + 5: msgs1 = so1.extract_received_messages() msgs2 = so2.extract_received_messages() if len(msgs1) == N + 1 and len(msgs2) == N + 1: break time.sleep(0.1) self.assertEqual(msgs1, msgs2)
def generic_test_two_clients_with_message_list(self, message_list): with make_client(endpoint=endpoint, appkey=appkey, reconnect_interval=0) as subscriber: with make_client(endpoint=endpoint, appkey=appkey) as publisher: co1 = ClientObserver() subscriber.observer = co1 co2 = ClientObserver() publisher.observer = co2 so = sync_subscribe(subscriber, channel) for msg in message_list: publisher.publish(channel, msg) sync_publish(publisher, channel, 'finalizer') while 'finalizer' !=\ so.last_received_channel_data['messages'][-1]: print(so.last_received_channel_data) last_data = so.wait_for_channel_data() if last_data['messages'][-1] == 'finalizer': break got_messages = so.extract_received_messages() self.assertEqual(got_messages, message_list + ['finalizer'])
def test_two_clients_with_deduplication(self): with make_client(endpoint=endpoint, appkey=appkey) as pub: with make_client(endpoint=endpoint, appkey=appkey) as sub: so = sync_subscribe(sub, channel, args={'only': 'value_changes'}) pub.publish(channel, "first") so.wait_for_channel_data() for _ in range(10): pub.publish(channel, "second") so.wait_for_channel_data() pub.publish(channel, "third") so.wait_for_channel_data() got_messages = so.extract_received_messages() self.assertEqual(got_messages, ['first', 'second', 'third'])
def main(): with make_client(endpoint=endpoint, appkey=application_key.appkey) as client: print('Connected to Satori RTM!') class SubscriptionObserver(object): def on_subscription_data(self, data): for message in data['messages']: if (message['data']['network'] == 'DASH'): try: producer.send(topic2, json.dumps(message)) producer.flush() except: producer.send(topic, json.dumps(message)) producer.flush() elif (message['data']['network'] == 'LTC' or message['data']['network'] == 'DOGE'): producer.send(topic2, json.dumps(message)) producer.flush() else: producer.send(topic, json.dumps(message)) producer.flush() subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) try: while True: time.sleep(1) except KeyboardInterrupt: pass
def main(): with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected to Satori RTM!') class SubscriptionObserver(object): # Called when the subscription is established. def on_enter_subscribed(self): print "Subscribed to the channel '{}' with send_fraction {}".format( channel, send_fraction) def on_enter_failed(self, reason): print 'Subscription failed, reason:', reason sys.exit(1) def on_subscription_data(self, data): count = 0 for i, message in enumerate(data['messages']): # send message to kafka if i % (1 / send_fraction) == 0: producer.send('weather', key='weather', value=message) count += 1 print 'sending {} msgs in batch'.format(count) subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) try: while True: time.sleep(1) pass except KeyboardInterrupt: pass
def test_channel_search(self): with make_client(endpoint=endpoint, appkey=appkey) as client: event = threading.Event() mailbox = [] def callback(ack): mailbox.append(ack) if ack['action'] != 'rtm/search/data': event.set() sync_publish(client, channel, 'ping') client.search('', callback=callback) event.wait(10) self.assertTrue(len(mailbox) > 0) last_pdu = mailbox[-1] self.assertEqual(last_pdu['action'], 'rtm/search/ok') for non_last_pdu in mailbox[:-1]: self.assertEqual(non_last_pdu['action'], 'rtm/search/data') result = [] for pdu in mailbox: result += pdu['body']['channels'] self.assertTrue(channel in result)
def test_wrong_subscribe_ack(self): with make_client(endpoint=endpoint, appkey=appkey, reconnect_interval=10) as client: client.observer = ClientObserver() old_received_message =\ client._internal.connection.on_incoming_json client._internal.connection.on_incoming_json =\ lambda *args: None so = SubscriptionObserver() client.subscribe('test_wrong_subscribe_ack', SubscriptionMode.ADVANCED, so) client._queue.join() old_received_message({ u"action": u"rtm/publish/ok", u"body": {}, u"id": 0 }) client._queue.join() expected_log = ['on_leave_connected', 'on_enter_awaiting'] self.assertEqual(client.observer.log[:2], expected_log)
def test_write_read(self): with make_client(endpoint, appkey) as client: k = make_channel_name('kv') v = make_channel_name('message') mailbox = [] event = Event() def callback(ack): mailbox.append(ack) event.set() client.write(k, v, callback=callback) event.wait(10) event.clear() client.read(k, callback=callback) event.wait(10) assert len(mailbox) == 2 write_ack = mailbox[0] read_ack = mailbox[1] assert write_ack['action'] == 'rtm/write/ok' assert read_ack['action'] == 'rtm/read/ok' assert read_ack['body']['message'] == v
def satori_client(): # connect to transportation data stream at satori with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected to channel: {}'.format(topic)) mailbox = [] got_message_event = threading.Event() class SubscriptionObserver(object): def on_subscription_data(self, data): for message in data['messages']: mailbox.append(message) got_message_event.set() # instantiate class subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) if not got_message_event.wait(20): print('Message timeout exceeded.') sys.exit(1) # return json in mailbox object while mailbox: send_json(json.dumps(mailbox[-1], sort_keys=True).encode('utf-8'))
def test_concurrent_publishes_in_different_clients(self): mailbox2 = [] self.publish_errors = [] def work(): with make_client(endpoint=endpoint, appkey=appkey) as publisher: for i in six.moves.range(100): try: sync_publish(publisher, channel, message) except Exception as e: self.publish_errors.append(e) with make_client(endpoint=endpoint, appkey=appkey) as subscriber: so = sync_subscribe(subscriber, channel) so.on_subscription_data = lambda data:\ mailbox2.extend(data['messages']) threads = [] for i in range(10): threads.append(threading.Thread(target=work)) for t in threads: t.start() for t in threads: t.join() time.sleep(3) self.assertEqual([], self.publish_errors) # assertEqual seems to choke on 1000 of 'hello's self.assertEqual(len(mailbox2), 1000) self.assertTrue(all((m == 'hello' for m in mailbox2)))
def test_concurrent_publishes_in_one_client(self): with make_client(endpoint=endpoint, appkey=appkey) as client: mailbox = [] self.publish_errors = [] so = sync_subscribe(client, channel) so.on_subscription_data = lambda data:\ mailbox.extend(data['messages']) def work(): for i in six.moves.range(100): try: sync_publish(client, channel, message) except Exception as e: self.publish_errors.append(e) threads = [] for i in range(10): threads.append(threading.Thread(target=work)) for t in threads: t.start() for t in threads: t.join() print('Waiting for the rest of messages') time.sleep(3) print("We've hopefully waited long enough") self.assertEqual([], self.publish_errors) # assertEqual seems to choke on 1000 of 'hello's self.assertEqual(len(mailbox), 1000) self.assertTrue(all((m == 'hello' for m in mailbox)))
def ask_price_ether(intent, session): with make_client(endpoint=endpoint, appkey=appkey) as client: dollars, cents = 0, 0 tick = [] event = threading.Event() def read_callback(reply): tick.append(reply) event.set() client.read(channel, callback=read_callback) if not event.wait(2): print('Read request timed out') else: price = tick[0]['body']['message']['price'] exchange = tick[0]['body']['message']['exchange'] dollars = int(price) cents = int((price - dollars)*100) #room for improvement, truncates for now instead of rounding fractional pennies session_attributes = {} reprompt_text = None speech_output = "Last Price was " + str(dollars) + \ " dollars and " + str(cents) + " cents, reported by the " + exchange + " exchange." should_end_session = False return build_response(session_attributes, build_speechlet_response( intent['name'], speech_output, reprompt_text, should_end_session))
def main(): with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected!') mailbox = [] got_message_event = threading.Event() class SubscriptionObserver(object): def on_subscription_data(self, data): for message in data['messages']: mailbox.append(message) got_message_event.set() subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) if not got_message_event.wait(10): print("Timeout while waiting for a message") sys.exit(1) while True: for message in mailbox: msg = json.dumps(message, ensure_ascii=False) producer.send(topic, msg.encode()) # do not send the messages to fast for development time.sleep(0.1)
def main(): with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected!') mailbox = [] got_message_event = threading.Event() class SubscriptionObserver(object): def on_subscription_data(self, data): for message in data['messages']: mailbox.append(message) #response = client1.put_record(DeliveryStreamName='traffic_stream',Record={'Data': json.dumps(message) + '\n'}) got_message_event.set() subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) if not got_message_event.wait(30): print("Timeout while waiting for a message") sys.exit(1) for message in mailbox: response = client1.put_record( DeliveryStreamName='traffic_stream', Record={'Data': json.dumps(message) + '\n'}) print(message)
def main(): with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected to Satori RTM!') class SubscriptionObserver(object): def on_subscription_data(self, data): ADDRESS = ("amqps://" "send" ":" "RJQLhw9Uu8qHL0KZQkb0dd+4didTvGqIzJZljtP5m/s=" "@" "structstreaming.servicebus.windows.net" "/" "twitter") sender = Sender() azClient = EventHubClient(ADDRESS if len(sys.argv) == 1 else sys.argv[1]) \ .publish(sender) \ .run_daemon() for message in data['messages']: sender.send(EventData(str(message))) print("Got message:", message) subscription_observer = SubscriptionObserver() client.subscribe(channel, SubscriptionMode.SIMPLE, subscription_observer) try: while True: time.sleep(1) except KeyboardInterrupt: pass
def test_exception_in_subscription_callback(self): with make_client(endpoint=endpoint, appkey=appkey) as client: exit = threading.Event() class CrashyObserver(SubscriptionObserver): def on_deleted(this): SubscriptionObserver.on_deleted(this) exit.set() raise ValueError('Error in on_deleted') def on_enter_subscribing(this): SubscriptionObserver.on_enter_subscribing(this) raise ValueError('Error in on_enter_subscribing') so = CrashyObserver() client.subscribe(channel, SubscriptionMode.ADVANCED, so) so.wait_subscribed() client.unsubscribe(channel) client.subscribe(channel, SubscriptionMode.ADVANCED, CrashyObserver()) client.unsubscribe(channel) client.subscribe(channel, SubscriptionMode.ADVANCED, SubscriptionObserver(), args={'position': '123'}) so.wait_deleted() client.unsubscribe(channel) if not exit.wait(20): raise RuntimeError('Timeout')
def work(): with make_client(endpoint=endpoint, appkey=appkey) as publisher: for i in six.moves.range(100): try: sync_publish(publisher, channel, message) except Exception as e: self.publish_errors.append(e)
def test_missing_subscription_observer_callbacks_are_fine(self): with make_client(endpoint=endpoint, appkey=appkey) as client: client.subscribe( make_channel_name('missing_subscription_callbacks'), SubscriptionMode.ADVANCED, object()) time.sleep(3)
def test_client_state_callbacks(self): with make_client(endpoint=endpoint, appkey=appkey) as client: global step step = 1 class TestClientObserver(ClientObserver): def on_enter_stopped(this): ClientObserver.on_enter_stopped(this) global step if step == 1: step = 2 client.start() elif step == 3: exit.set() def on_enter_connected(this): ClientObserver.on_enter_connected(this) global step if step == 2: step = 3 client.stop() co = TestClientObserver() client.observer = co exit = threading.Event() stop_timer = threading.Timer(0.01, lambda: client.stop()) stop_timer.daemon = True stop_timer.name = 'StopTimer' stop_timer.start() if not exit.wait(20): raise RuntimeError('Timeout')
def test_write_delete_read(self): with make_client(endpoint, appkey) as client: k = make_channel_name('delete_read') v = make_channel_name('value') mailbox = [] event = Event() def callback(ack): mailbox.append(ack) event.set() client.write(k, v, callback=callback) event.wait(10) event.clear() client.delete(k, callback=callback) event.wait(10) event.clear() client.read(k, callback=callback) event.wait(10) assert len(mailbox) == 3 assert mailbox[0]['action'] == 'rtm/write/ok' assert mailbox[1]['action'] == 'rtm/delete/ok' assert mailbox[2]['action'] == 'rtm/read/ok' assert mailbox[2]['body']['message'] is None
def test_publish_to_restricted_channel(self): with make_client(endpoint=endpoint, appkey=appkey) as client: auth_event = threading.Event() auth_delegate = auth.RoleSecretAuthDelegate(role, secret) mailbox = [] def auth_callback(auth_result): if type(auth_result) == auth.Done: mailbox.append('Auth success') auth_event.set() else: mailbox.append('Auth failure: {0}'.format(auth_result)) auth_event.set() client.authenticate(auth_delegate, auth_callback) if not auth_event.wait(60): raise RuntimeError('Auth never finished') if not mailbox == ['Auth success']: raise RuntimeError(mailbox) sync_publish(client, channel, 'ohai')
def test_quickly_change_observer_twice(self): with make_client(endpoint=endpoint, appkey=appkey) as client: channel = make_channel_name('change_observer_twice') so2 = SubscriptionObserver() so1 = sync_subscribe(client, channel) client.unsubscribe(channel) client.subscribe(channel, SubscriptionMode.ADVANCED, so2) client.unsubscribe(channel) client.subscribe(channel, SubscriptionMode.ADVANCED, None) client.unsubscribe(channel) so3 = sync_subscribe(client, channel) expected_so1_log = [ 'on_leave_unsubscribed', 'on_enter_subscribing', 'on_leave_subscribing', 'on_enter_subscribed', 'on_leave_subscribed', 'on_enter_unsubscribing', 'on_leave_unsubscribing', 'on_enter_unsubscribed', 'on_deleted' ] expected_so2_log = ['on_deleted'] expected_so3_log = [ 'on_created', 'on_leave_unsubscribed', 'on_enter_subscribing', 'on_leave_subscribing', 'on_enter_subscribed' ] self.assertEqual(so1.log, expected_so1_log) self.assertEqual(so2.log, expected_so2_log) self.assertEqual(so3.log, expected_so3_log)
def test_double_unsubscribe_error(self): with make_client( endpoint=endpoint, appkey=appkey) as client: client.unsubscribe(channel) so = sync_subscribe(client, channel) client.unsubscribe(channel) so.wait_deleted() expected_log = [ 'on_leave_unsubscribed', 'on_enter_subscribing', 'on_leave_subscribing', 'on_enter_subscribed', 'on_leave_subscribed', 'on_enter_unsubscribing', 'on_leave_unsubscribing', 'on_enter_unsubscribed', 'on_deleted' ] self.assertEqual(so.log, expected_log) self.assertTrue( channel not in client._internal.subscriptions) client.unsubscribe(channel)
def main(): with make_client( endpoint=endpoint, appkey=appkey) as client: print('Connected!') mailbox = [] got_message_event = threading.Event() class SubscriptionObserver(object): def on_subscription_data(self, data): for message in data['messages']: mailbox.append(message) got_message_event.set() subscription_observer = SubscriptionObserver() client.subscribe( channel, SubscriptionMode.SIMPLE, subscription_observer) if not got_message_event.wait(10): print("Timeout while waiting for a message") sys.exit(1) for message in mailbox: print('Got message "{0}"'.format(message))
def test_unsubscribe_nack_error(self): with make_client( endpoint=endpoint, appkey=appkey) as client: client.unsubscribe(channel) so = sync_subscribe(client, channel) old_received_message =\ client._internal.connection.on_incoming_text_frame client._internal.connection.on_incoming_text_frame =\ lambda *args: None client.unsubscribe(channel) client._queue.join() old_received_message( b'{"action":"rtm/unsubscribe/error","body":{},"id":1}') client._queue.join() expected_log = [ 'on_leave_unsubscribed', 'on_enter_subscribing', 'on_leave_subscribing', 'on_enter_subscribed', 'on_leave_subscribed', 'on_enter_unsubscribing', 'on_leave_unsubscribing', 'on_enter_subscribed'] self.assertEqual(so.log, expected_log)
def test_wrong_unsubscribe_ack(self): with make_client(endpoint=endpoint, appkey=appkey) as client: client.observer = ClientObserver() sync_subscribe(client, channel) old_received_message =\ client._internal.connection.on_incoming_text_frame client._internal.connection.on_incoming_text_frame =\ lambda *args: None client.unsubscribe(channel) client._queue.join() old_received_message( b'{"action":"rtm/publish/ok","body":{},"id":1}') client._queue.join() client.observer.wait_connected() expected_log = [ 'on_leave_connected', 'on_enter_awaiting', 'on_leave_awaiting', 'on_enter_connecting', 'on_leave_connecting', 'on_enter_connected', ] self.assertEqual(client.observer.log, expected_log)
def bad_connect(): with make_client(endpoint='ws://qwerty:9999', appkey=appkey, reconnect_interval=0, fail_count_threshold=2): pass
def main(): import logging logging.basicConfig(level=logging.WARNING) class SubscriptionObserver(object): def on_enter_subscribed(self): print('Subscribed to: animals') def on_subscription_data(self, pdu): for animal in pdu['messages']: print('Got animal {0}: {1}'.format(animal['who'], animal)) def on_enter_failed(self, reason): print('Subscription failed:', reason, file=sys.stderr) with make_client(endpoint=endpoint, appkey=appkey) as client: print('Connected to Satori RTM!') observer = SubscriptionObserver() client.subscribe(u'animals', SubscriptionMode.SIMPLE, observer) print('Press CTRL-C to exit', file=sys.stderr) try: while True: time.sleep(1) except KeyboardInterrupt: pass
def test_channel_error_during_unsubscribing(self): with make_client(endpoint=endpoint, appkey=appkey) as client: so = sync_subscribe(client, channel) client._internal.subscriptions[channel]\ .on_unsubscribe_ok = lambda *args: None client.unsubscribe(channel) client._queue.join() emulate_channel_error(client, channel) so.wait_not_subscribed() sync_publish(client, channel, u'should-be-missed') so.wait_not_subscribed() expected_log = [ 'on_leave_unsubscribed', 'on_enter_subscribing', 'on_leave_subscribing', 'on_enter_subscribed', 'on_leave_subscribed', 'on_enter_unsubscribing', 'on_leave_unsubscribing', 'on_enter_unsubscribed', 'on_deleted' ] self.assertEqual(so.log, expected_log)
def test_two_clients_with_best_effort_delivery(self): with make_client(endpoint=endpoint, appkey=appkey, reconnect_interval=0) as subscriber: with make_client(endpoint=endpoint, appkey=appkey) as publisher: co1 = ClientObserver() subscriber.observer = co1 co2 = ClientObserver() publisher.observer = co2 try: so = sync_subscribe(subscriber, channel, mode=SubscriptionMode.RELIABLE) sync_publish(publisher, channel, 'first-message') so.wait_for_channel_data('First receive timeout') emulate_websocket_disconnect(subscriber) so.wait_not_subscribed() # send a message while subscriber is disconnected sync_publish(publisher, channel, 'second-message') so.wait_subscribed('Second subscribe timeout') so.wait_for_channel_data('Second receive timeout') # send a message after subscribed reconnected publisher.publish(channel, 'third-message') so.wait_for_channel_data('Third receive timeout') expected_messages =\ ['first-message', 'second-message', 'third-message'] got_messages = [] for log_entry in so.log: if log_entry[0] == 'data': got_messages += log_entry[1]['messages'] self.assertEqual(got_messages, expected_messages) except Exception: print('Subscriber log: {0}'.format(co1.log)) print('Publisher log: {0}'.format(co2.log)) print('Subscription log: {0}'.format(so.log)) raise