def writeMetric(metricList, host, port, username, password, vhost, exchange, queue, spec=None, channel_number=1, ssl=False): global conn if not spec: spec = txamqp.spec.load(os.path.normpath( os.path.join(os.path.dirname(__file__), 'amqp0-8.xml'))) delegate = TwistedDelegate() connector = ClientCreator(reactor, AMQClient, delegate=delegate, vhost=vhost, spec=spec) if ssl: from twisted.internet.ssl import ClientContextFactory conn = yield connector.connectSSL(host, port, ClientContextFactory()) else: conn = yield connector.connectTCP(host, port, timeout=130) yield conn.authenticate(username, password) channel = yield conn.channel(channel_number) yield channel.channel_open() yield channel.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False) #reply = yield channel.queue_declare(queue = queue, durable = True) #my_queue = reply.queue #Pickup settings.BIND_PATTERNS somewhere else #for bind_pattern in settings.BIND_PATTERNS: # yield channel.queue_bind(exchange=exchange, queue=my_queue, routing_key=bind_pattern) for (metric, datapoints) in metricList: body = "" for point in datapoints: temp = "%f %d\n"%(point[1], point[0]) body = body + temp message = Content(body) message["delivery mode"] = 2 channel.basic_publish(exchange=exchange, content=message, routing_key=metric) yield channel.channel_close() yield conn.close("Publish Done. Closing Connection.")
def test_fragment_body(self): """ Body frames must be fragmented according to the broker frame_max parameter. """ channel = self.channel yield channel.queue_declare(queue="test-get", exclusive=True) large_string = ''.join('x' for _ in range(self.client.MAX_LENGTH * 2)) msg = Content(large_string) msg["delivery mode"] = 2 channel.basic_publish(routing_key="test-get", content=msg) reply = yield channel.basic_get(no_ack=True) self.assertEqual(reply.method.klass.name, "basic") self.assertEqual(reply.method.name, "get-ok") self.assertEqual(large_string, reply.content.body)
def test_qos_prefetch_count(self): """ Test that the prefetch count specified is honoured """ # setup: declare queue and subscribe channel = self.channel # set prefetch to 5: yield channel.basic_qos(prefetch_count=5) yield channel.queue_declare(queue="test-prefetch-count", exclusive=True) subscription = yield channel.basic_consume(queue="test-prefetch-count", no_ack=False) queue = yield self.client.queue(subscription.consumer_tag) # publish 10 messages: for i in range(1, 11): channel.basic_publish(routing_key="test-prefetch-count", content=Content("Message %d" % i)) # only 5 messages should have been delivered: for i in range(1, 6): msg = yield queue.get(timeout=1) self.assertEqual("Message %d" % i, msg.content.body) try: extra = yield queue.get(timeout=1) self.fail("Got unexpected 6th message in original queue: " + extra.content.body) except Empty: pass # ack messages and check that the next set arrive ok: channel.basic_ack(delivery_tag=msg.delivery_tag, multiple=True) for i in range(6, 11): msg = yield queue.get(timeout=1) self.assertEqual("Message %d" % i, msg.content.body) channel.basic_ack(delivery_tag=msg.delivery_tag, multiple=True) try: extra = yield queue.get(timeout=1) self.fail("Got unexpected 11th message in original queue: " + extra.content.body) except Empty: pass
def call(n): corr_id = str(uuid.uuid4()) reply = yield channel.queue_declare(exclusive=True) callback_queue = reply.queue msg = Content(str(n)) msg['correlation id'] = corr_id msg['reply to'] = callback_queue yield channel.basic_publish(exchange='', routing_key='rpc_queue', content=msg) print ' [x] Sent "%s"' % n yield channel.basic_consume(queue=callback_queue, no_ack=True, consumer_tag='qtag') queue = yield connection.queue('qtag') while True: response = yield queue.get() if response.content.properties['correlation id'] == corr_id: returnValue(response.content.body)
def test_get_concurrent_with_error(self): """ If an error occurs in a call to get(), other calls don't fail, and are retried on reconnection instead. """ client1 = yield self.service.whenConnected() deferred = self.source.get("uuid", 0) with self.assertRaises(NotFound): yield self.source.get("uuid-unknown", 0) yield self.channel.basic_publish(routing_key="uuid", content=Content("hello")) notification = yield deferred self.assertEqual("hello", notification.payload) client2 = yield self.service.whenConnected() # The ClientService has reconnected, yielding a new client. self.assertIsNot(client1, client2)
def publish4(result, message, count_): connection, channel = result exchange_name = '%s_4' % EXCHANGE_NAME yield channel.exchange_declare(exchange=exchange_name, type='direct') for i in range(count_): msg = Content('%s [%04d]' % ( message, i, )) severity = SEVERITIES[i % len(SEVERITIES)] yield channel.basic_publish(exchange=exchange_name, routing_key=severity, content=msg) print ' [x] Sent "%s [%04d]" with severity [%s]' % ( message, i, severity, ) returnValue(result)
def read_content(queue): frame = yield queue.get() header = frame.payload children = [] for i in range(header.weight): content = yield read_content(queue) children.append(content) size = header.size read = 0 buf = BytesIO() while read < size: body = yield queue.get() content = body.payload.content if isinstance(content, str): content = content.encode() buf.write(content) read += len(content) defer.returnValue(Content(buf.getvalue(), children, header.properties.copy()))
def test_channel_flow(self): channel = self.channel yield channel.queue_declare(queue="flow_test_queue", exclusive=True) yield channel.basic_consume(consumer_tag="my-tag", queue="flow_test_queue") incoming = yield self.client.queue("my-tag") yield channel.channel_flow(active=False) channel.basic_publish(routing_key="flow_test_queue", content=Content("abcdefghijklmnopqrstuvwxyz")) try: yield incoming.get(timeout=1) self.fail("Received message when flow turned off.") except Empty: None yield channel.channel_flow(active=True) msg = yield incoming.get(timeout=1) self.assertEqual("abcdefghijklmnopqrstuvwxyz", msg.content.body)
def writeMetric(metric_path, value, timestamp, host, port, username, password, vhost, exchange, spec=None, channel_number=1): if not spec: spec = txamqp.spec.load( os.path.normpath( os.path.join(os.path.dirname(__file__), 'amqp0-8.xml'))) delegate = TwistedDelegate() connector = ClientCreator(reactor, AMQClient, delegate=delegate, vhost=vhost, spec=spec) conn = yield connector.connectTCP(host, port) yield conn.authenticate(username, password) channel = yield conn.channel(channel_number) yield channel.channel_open() yield channel.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False) message = Content("%f %d" % (value, timestamp)) message["delivery mode"] = 2 channel.basic_publish(exchange=exchange, content=message, routing_key=metric_path) yield channel.channel_close()
def test_start_consuming_later(self): """Related to #67, will start consuming after publishing messages, this will imitate starting a connector with some pending messages for it""" yield self.connect() yield self.amqp.chan.exchange_declare(exchange='messaging', type='topic', durable='true', arguments=arguments) # Consume consumerTag = 'lateConsumerTest-%s' % (str(uuid.uuid4())) yield self.amqp.named_queue_declare(queue="submit.sm.connector01", durable='true', arguments=arguments) yield self.amqp.chan.queue_bind(queue="submit.sm.connector01", exchange="messaging", routing_key="submit.sm.*") yield self.amqp.chan.basic_consume(queue="submit.sm.connector01", no_ack=False, consumer_tag=consumerTag) queue = yield self.amqp.client.queue(consumerTag) # Publish for i in range(5000): yield self.amqp.publish(exchange='messaging', routing_key="submit.sm.connector01", content=Content(str(i))) # Start consuming (same as starting a connector) queue.get().addCallback(self._callback, queue, ack=True).addErrback(self._errback) # Wait for 15 seconds # (give some time to the consumer to get its work done) yield waitFor(15) yield queue.close() yield self.amqp.disconnect() self.assertEqual(self.consumedMessages, 5000)
def test_get_properties_with_unicode(self): """ Test basic_get method with a property having unicode content Refs #4 """ channel = self.channel yield channel.queue_declare(queue="test-get-header-unicode", exclusive=True) # publish a message (no_ack=True) with persistent messaging msg = Content("some value") msg["delivery mode"] = 2 msg['headers'] = {"somekey": u"any thing in unicode"} channel.basic_publish(routing_key="test-get-header-unicode", content=msg) # use basic_get to read back the messages, and check that we get an empty at the end reply = yield channel.basic_get(no_ack=True) self.assertEqual(reply.method.klass.name, "basic") self.assertEqual(reply.method.name, "get-ok") self.assertEqual("some value", reply.content.body) self.assertEqual({"somekey": u"any thing in unicode"}, reply.content.properties['headers'])
def sendEvent(self, event): """ Write an event to Logstash. """ if not self.chan: log.msg("No AMQP channel. Dropping event.") return event = copy.copy(event) # Pass timestamp on as a string, to work around Logstash bug 279. event['timestamp'] = repr(event['timestamp']) # Make sure isError is always a boolean if 'isError' in event: event['isError'] = bool(event['isError']) body = simplejson.dumps(event) content = Content(body) return self.chan.basic_publish(exchange=self.exchange, content=content)
def readContent(queue): frame = queue.get() yield frame header = frame.payload children = [] for i in range(header.weight): content = readContent(queue) yield content children.append(content) size = header.size read = 0 buf = StringIO() while read < size: body = queue.get() yield body content = body.payload.content buf.write(content) read += len(content) defer.returnValue( Content(buf.getvalue(), children, header.properties.copy()))
def send_message(self, exchange, routing_key, msg, mandatory=False, headers=None, declareExchange=True): body = msg headers = headers if headers else {} # it is a protobuf if not isinstance(msg, basestring): body = msg.SerializeToString() headers['X-Protobuf-FullName'] = msg.DESCRIPTOR.full_name queueSchema = self.factory.queueSchema exchangeConfig = queueSchema.getExchange(exchange) if declareExchange: # Declare the exchange to which the message is being sent yield getAdapter( self.chan, IAMQPChannelAdapter).declareExchange(exchangeConfig) if exchangeConfig.compression == 'deflate': body = zlib.compress(body) content = Content(body) content['delivery_mode'] = exchangeConfig.delivery_mode # set the headers to our protobuf type, hopefully this works content.properties['headers'] = headers content.properties['content-type'] = 'application/x-protobuf' if exchangeConfig.compression == 'deflate': content.properties['content-encoding'] = 'deflate' # Publish away yield self.chan.basic_publish(exchange=exchangeConfig.name, routing_key=routing_key, content=content, mandatory=mandatory) returnValue("SUCCESS")
def test_basic_delivery_queued(self): """ Test basic message delivery where publish is issued before consume (i.e. requires queueing of the message) """ channel = self.channel yield self.exchange_declare(channel, exchange="test-exchange", type="direct") yield self.queue_declare(channel, queue="test-queue") yield channel.queue_bind(queue="test-queue", exchange="test-exchange", routing_key="key") body = "Queued Delivery" channel.basic_publish(exchange="test-exchange", routing_key="key", content=Content(body)) reply = yield channel.basic_consume(queue="test-queue", no_ack=True) queue = yield self.client.queue(reply.consumer_tag) msg = yield queue.get(timeout=5) self.assert_(msg.content.body == body)
def test_consume_all_requeued_messages(self): "Related to #67, test for consuming all requeued messages" yield self.connect() yield self.amqp.chan.exchange_declare(exchange='messaging', type='topic', durable='true', arguments=arguments) # Consume yield self.amqp.named_queue_declare(queue="submit.sm_all_1", durable='true', arguments=arguments) yield self.amqp.chan.queue_bind(queue="submit.sm_all_1", exchange="messaging", routing_key="submit.sm.*") yield self.amqp.chan.basic_consume(queue="submit.sm_all_1", no_ack=False, consumer_tag='qtag') queue = yield self.amqp.client.queue('qtag') queue.get().addCallback(self._callback_reject_once, queue, reject=True).addErrback(self._errback) # Publish for i in range(3000): yield self.amqp.publish(exchange='messaging', routing_key="submit.sm.connector01", content=Content(str(i))) # Wait for 20 seconds # (give some time to the consumer to get its work done) yield waitFor(20) yield queue.close() yield self.amqp.disconnect() self.assertEqual(self.rejectedMessages, 3000) self.assertEqual(self.consumedMessages, 3000)
def send_message(self, exchange, routing_key, msg, mandatory=False, headers=None, declareExchange=True): body = msg headers = headers if headers else {} # it is a protobuf if not isinstance(msg, basestring): body = msg.SerializeToString() headers['X-Protobuf-FullName'] = msg.DESCRIPTOR.full_name queueSchema = self.factory.queueSchema exchangeConfig = queueSchema.getExchange(exchange) if declareExchange: # Declare the exchange to which the message is being sent yield getAdapter(self.chan, IAMQPChannelAdapter).declareExchange(exchangeConfig) if exchangeConfig.compression == 'deflate': body = zlib.compress(body) content = Content(body) content.properties['delivery-mode'] = exchangeConfig.delivery_mode content.properties['headers'] = headers content.properties['content-type'] = 'application/x-protobuf' if exchangeConfig.compression == 'deflate': content.properties['content-encoding'] = 'deflate' # Publish away try: yield self.chan.basic_publish(exchange=exchangeConfig.name, routing_key=routing_key, content=content, mandatory=mandatory) except ClientClosed as e: log.warn("Caught txamqp.client.Closed Exception. Trying to recover the channel.") self.channelFailed(self.chan, e.message) returnValue("SUCCESS")
def test_simple_publish_consume(self): yield self.connect() yield self.amqp.named_queue_declare(queue="submit.sm.connector01") # Consume yield self.amqp.chan.basic_consume(queue="submit.sm.connector01", no_ack=True, consumer_tag='qtag') queue = yield self.amqp.client.queue('qtag') queue.get().addCallback(self._callback, queue).addErrback(self._errback) # Publish yield self.amqp.publish(routing_key="submit.sm.connector01", content=Content(self.message)) # Wait for 2 seconds # (give some time to the consumer to get its work done) yield waitFor(2) yield queue.close() yield self.amqp.disconnect() self.assertEqual(self.consumedMessages, 1)
def test_publish_pickled_binary_content(self): """Refs #640 As of 12/5/2017 txamqp 0.8 were released and it caused an outage to Jasmin. """ yield self.connect() yield self.amqp.chan.exchange_declare(exchange='messaging', type='topic', durable='true', arguments=arguments) # Consume yield self.amqp.named_queue_declare(queue="submit.sm_all", durable='true', arguments=arguments) yield self.amqp.chan.queue_bind(queue="submit.sm_all", exchange="messaging", routing_key="submit.sm.*") yield self.amqp.chan.basic_consume(queue="submit.sm_all", no_ack=True, consumer_tag='qtag') queue = yield self.amqp.client.queue('qtag') queue.get().addCallback(self._callback, queue).addErrback(self._errback) # Publish a pickled binary content with v2 protocol yield self.amqp.publish(exchange='messaging', routing_key="submit.sm.connector01", content=Content(pickle.dumps('\x53', 2))) # Wait for 2 seconds # (give some time to the consumer to get its work done) yield waitFor(2) yield queue.close() yield self.amqp.disconnect() self.assertEqual(self.consumedMessages, 1)
def _send_message(self, exchange, routing_key, msg, deferred, type="direct", durable=True, auto_delete=False): """Send a single message.""" # First declare the exchange just in case it doesn't exist. yield self.chan.exchange_declare(exchange=exchange, type=type, durable=durable, auto_delete=auto_delete) msg = Content(msg) msg["delivery mode"] = 2 # 2 = persistent delivery. d = self.chan.basic_publish(exchange=exchange, routing_key=routing_key, content=msg) d.addErrback(self._send_message_err) d.chainDeferred(deferred)
def test_wb_get_with_broker_shutdown_during_message_wait(self): """ If rabbitmq gets shutdown while we wait for messages, we transparently wait for the reconnection and try again. """ # This will make the connector setup the channel before we call # get(), so by the time we call it in the next line all # connector-related deferreds will fire synchronously and the # code will block on basic-consume. yield self.connector() d = self.source.get("uuid", 0) # Acquiring the channel lock makes sure that basic-consume has # succeeded and we started waiting for the message. yield self.source._channel_lock.acquire() self.source._channel_lock.release() # Restart rabbitmq yield self.client.close() yield self.client.disconnected.wait() self.rabbit.cleanUp() self.rabbit.config = RabbitServerResources( port=self.rabbit.config.port) # Ensure that we use the same port self.rabbit.setUp() # Get a new channel and re-declare the queue, since the restart has # destroyed it. self.client = yield self.endpoint.connect(self.factory) self.channel = yield self.client.channel(1) yield self.channel.channel_open() yield self.channel.queue_declare(queue="uuid") # Publish a message in the queue yield self.channel.basic_publish(routing_key="uuid", content=Content("hello")) notification = yield d self.assertEqual("hello", notification.payload)
def invoke(self, method, args, content=None): if self.closed: raise Closed(self.reason) frame = Frame(self.id, Method(method, *args)) self.outgoing.put(frame) if method.content: if content == None: content = Content() self.writeContent(method.klass, content, self.outgoing) try: # here we depend on all nowait fields being named nowait f = method.fields.byname["nowait"] nowait = args[method.fields.index(f)] except KeyError: nowait = False try: if not nowait and method.responses: resp = self.responses.get() yield resp resp = resp.payload if resp.method.content: content = readContent(self.responses) yield content else: content = None if resp.method in method.responses: defer.returnValue(Message(resp.method, resp.args, content)) else: raise ValueError(resp) except QueueClosed, e: if self.closed: raise Closed(self.reason) else: raise e
def gotConnection(conn, username, password, body, count=1): print("Connected to broker.") yield conn.authenticate(username, password) print("Authenticated. Ready to send messages") chan = yield conn.channel(1) yield chan.channel_open() def send_messages(): def message_iterator(): for i in range(count): content = body + "-%d" % i msg = Content(content) msg["delivery mode"] = 2 chan.basic_publish(exchange="chatservice", content=msg, routing_key="txamqp_chatroom") print("Sending message: %s" % content) yield None return task.coiterate(message_iterator()) yield send_messages() stopToken = "STOP" msg = Content(stopToken) msg["delivery mode"] = 2 chan.basic_publish(exchange="chatservice", content=msg, routing_key="txamqp_chatroom") print("Sending message: %s" % stopToken) yield chan.channel_close() chan0 = yield conn.channel(0) yield chan0.connection_close() reactor.stop()
def test_delete_ifempty(self): """ Test that if_empty field of queue_delete is honoured """ channel = self.channel # create a queue and add a message to it (use default binding): yield channel.queue_declare(queue="delete-me-2") yield channel.queue_declare(queue="delete-me-2", passive="True") channel.basic_publish(routing_key="delete-me-2", content=Content("message")) # try to delete, but only if empty: try: yield channel.queue_delete(queue="delete-me-2", if_empty="True") self.fail("Expected delete if_empty to fail for non-empty queue") except Closed as e: self.assertChannelException(406, e.args[0]) # need new channel now: channel = yield self.client.channel(2) yield channel.channel_open() # empty queue: reply = yield channel.basic_consume(queue="delete-me-2", no_ack=True) queue = yield self.client.queue(reply.consumer_tag) msg = yield queue.get(timeout=1) self.assertEqual(b"message", msg.content.body) yield channel.basic_cancel(consumer_tag=reply.consumer_tag) # retry deletion on empty queue: yield channel.queue_delete(queue="delete-me-2", if_empty="True") # check that it has gone by declaring passively: try: yield channel.queue_declare(queue="delete-me-2", passive="True") self.fail("Queue has not been deleted") except Closed as e: self.assertChannelException(404, e.args[0])
def send_message(self, route_key, instructions): """ Compose a provisioning message and deliver it to an exchange with routing key `route_key`. """ log = self.log exchange = self.pub_exchange serialized = instructions.serialize() msg = Content(serialized) msg["delivery-mode"] = 2 success = False reconnect = False delay_time = 20 while not success: try: if reconnect: log.INFO("Attempting to recconect to publisher exchange ...") yield self.connect_to_exchange() channel = self.pub_channel channel.basic_publish( exchange=exchange, content=msg, routing_key=route_key) success = True reconnect = False except Exception as ex: log.warn("Error attempting to publish message: {error}", error=ex) log.warn("Will attempt to reconnect.") yield delay(self.reactor, delay_time) reconnect = True log.debug( "Sent message to target exchange '{exchange}' with routing key '{route_key}'.", event_type="amqp_send", exchange=exchange, route_key=route_key) log.debug("Send message: {msg}", event_type="amqp_send_msg", msg=serialized)
def test_get_binary(self): """ Test basic_get method Refs #4 """ channel = self.channel yield channel.queue_declare(queue="test-get-bin", exclusive=True) # publish some messages (no_ack=True) with persistent messaging for i in range(1, 11): msg = "Message %d" % i msg_binary = Content(pickle.dumps(msg, 2)) # msg_binary = Content(msg) msg_binary["delivery mode"] = 2 channel.basic_publish(routing_key="test-get-bin", content=msg_binary) # use basic_get to read back the messages, and check that we get an empty at the end for i in range(1, 11): reply = yield channel.basic_get(no_ack=True) self.assertEqual(reply.method.klass.name, "basic") self.assertEqual(reply.method.name, "get-ok") self.assertEqual("Message %d" % i, pickle.loads(reply.content.body))
def test_simple_publish_consume_by_topic(self): yield self.connect() yield self.amqp.chan.exchange_declare(exchange='messaging', type='topic', durable='true', arguments=arguments) # Consume yield self.amqp.named_queue_declare(queue="submit.sm_all", durable='true', arguments=arguments) yield self.amqp.chan.queue_bind(queue="submit.sm_all", exchange="messaging", routing_key="submit.sm.*") yield self.amqp.chan.basic_consume(queue="submit.sm_all", no_ack=True, consumer_tag='qtag') queue = yield self.amqp.client.queue('qtag') queue.get().addCallback(self._callback, queue).addErrback(self._errback) # Publish yield self.amqp.publish(exchange='messaging', routing_key="submit.sm.connector01", content=Content(self.message)) # Wait for 2 seconds # (give some time to the consumer to get its work done) yield waitFor(2) yield queue.close() yield self.amqp.disconnect() self.assertEqual(self.consumedMessages, 1)
def read_content(queue): frame = yield queue.get() header = frame.payload children = [] for i in range(header.weight): content = yield read_content(queue) children.append(content) size = header.size read = 0 buf = six.StringIO() while read < size: body = yield queue.get() content = body.payload.content # if this is the first instance of real binary content, convert the string buffer to BytesIO # Not a nice fix but it preserves the original behaviour if six.PY3 and isinstance(content, bytes) and isinstance( buf, six.StringIO): buf = six.BytesIO() buf.write(content) read += len(content) defer.returnValue( Content(buf.getvalue(), children, header.properties.copy()))
def test_ack_message(self): """ C{ack_message} confirms the removal of a message from the queue, making subsequent C{get_message} calls waiting for another message. """ yield self.manager.connected((self.client, self.channel)) yield self.channel.queue_declare(queue=self.queue_prefix + "uuid1") content = Content("some content") yield self.channel.basic_publish(routing_key=self.queue_prefix + "uuid1", content=content) message, tag = yield self.manager.get_message("uuid1", "0") yield self.manager.ack_message(tag) reply = yield self.client.queue(self.tag_prefix + "uuid1.1") reply.clock = self.clock event_queue = QueueWrapper(reply).event_queue d = self.manager.get_message("uuid1", "1") yield event_queue.get() yield deferLater(reactor, 0, lambda: None) self.clock.advance(self.manager.message_timeout + 1) yield assert_fails_with(d, Timeout)
def onPacket(self, response): self._update_work_defer() # types of messages if 'reply to' in response.content.properties: # call request reply = Content(self.onCall(response)) reply["delivery mode"] = 1 # non-persistent reply['correlation id'] = response.content.properties[ 'correlation id'] self.channel_rx.basic_publish(exchange='', routing_key=reply['reply to'], content=reply) return if 'correlation id' in response.content.properties: # call response if response.content.properties['correlation id'] in self.replies: self.replies[ response.content.properties['correlation id']].callback( response.content.body) return # just async message self.onRecv(response) return