def _build_basic_properties(self, properties): """ Get the pika.BasicProperties from a pikachewie.data.Properties object. :param pikachewie.data.Properties properties: properties to convert :rtype: pika.spec.BasicProperties """ basic_properties = BasicProperties() attrs = ''' app_id content_encoding content_type correlation_id delivery_mode priority reply_to message_id type user_id '''.split() for attr in attrs: value = getattr(properties, attr) if value is not None: setattr(basic_properties, attr, value) basic_properties.timestamp = properties.timestamp or int(time.time()) if properties.expiration is not None: basic_properties.expiration = str(properties.expiration) if properties.headers is not None and \ len(properties.headers.keys()): basic_properties.headers = dict(properties.headers) return basic_properties
def basic_publish(self, exchange, routing_key, message, properties=None, mandatory=True, immediate=False): ''' Publish a message to an exchange. Arguments: - `exchange` - The exchange to publish to. - `routing_key` - The routing key to bind on. - `message` - The message to send. If this is not a string it will be serialized as JSON, and the properties.content_type field will be forced to be application/json. - `properties` - Dict of AMQP message properties. - `mandatory` - AMQP Mandatory flag. - `immediate` - AMQP Immediate flag. ''' properties = BasicProperties(properties or {}) if not isinstance(message, basestring): # Convert to JSON string if it's not a string message = json.dumps(to_serializable(message)) properties.content_type = "application/json" if not isinstance(routing_key, basestring): raise InvalidRoutingKeyException("'%s' is not a valid routing key!" % routing_key) return self.channel.basic_publish(exchange, routing_key, message, properties=properties, mandatory=mandatory, immediate=immediate)
def decorated_callback( channel: Channel, method: Basic.Deliver, properties: BasicProperties, body: bytes, ) -> Any: if not properties: properties = BasicProperties(headers={}) if properties.headers is None: properties.headers = {} ctx = propagate.extract(properties.headers, getter=_pika_getter) if not ctx: ctx = context.get_current() token = context.attach(ctx) span = _get_span( tracer, channel, properties, destination=method.exchange if method.exchange else method.routing_key, span_kind=SpanKind.CONSUMER, task_name=task_name, operation=MessagingOperationValues.RECEIVE, ) try: with trace.use_span(span, end_on_exit=True): try: consume_hook(span, body, properties) except Exception as hook_exception: # pylint: disable=W0703 _LOG.exception(hook_exception) retval = callback(channel, method, properties, body) finally: context.detach(token) return retval
def decorated_function( exchange: str, routing_key: str, body: bytes, properties: BasicProperties = None, mandatory: bool = False, ) -> Any: if not properties: properties = BasicProperties(headers={}) if properties.headers is None: properties.headers = {} span = _get_span( tracer, channel, properties, destination=exchange if exchange else routing_key, span_kind=SpanKind.PRODUCER, task_name="(temporary)", operation=None, ) if not span: return original_function(exchange, routing_key, body, properties, mandatory) with trace.use_span(span, end_on_exit=True): if span.is_recording(): propagate.inject(properties.headers) try: publish_hook(span, body, properties) except Exception as hook_exception: # pylint: disable=W0703 _LOG.exception(hook_exception) retval = original_function(exchange, routing_key, body, properties, mandatory) return retval
def publish_message(self, item): """ Sends an AMQP message """ fields = item["fields"] priority = item["priority"] logger.debug("Protocol:publish_message - {client_id} - {priority} - {item}", client_id=self.factory.AMQPClient.client_id, priority=priority, item=item) try: message_meta = fields["message_meta"] fields["properties"] = BasicProperties(**fields["properties"]) fields["properties"].headers["msg_sent_at"] = str(time()) # This was not found for sslcert - was published before connection resquest! Find out why. yield self._channel.basic_publish(exchange=fields["exchange_name"], routing_key=fields["routing_key"], body=fields["body"], properties=fields["properties"]) message_meta["msg_sent_at"] = float(time()) message_meta["send_success"] = True except Exception as error: logger.warn("--------==(Error: While sending message. )==--------") logger.warn("--------------------------------------------------------") logger.warn("{error}", error=sys.exc_info()) logger.warn("---------------==(Traceback)==--------------------------") logger.warn("{trace}", trace=traceback.print_exc(file=sys.stdout)) logger.warn("--------------------------------------------------------") message_meta["send_success"] = False
def test_zvol_unmapped_finished_when_have_pull_scheduled(self, mock_run_command): zvol = 'vol4_busy' target = 'iqn.2001-04.com.nas-0-1-%s'%zvol bdev = 'sdc' mock_run_command.return_value = (iscsiadm_session_response%(target, zvol)).splitlines() with sqlite3.connect(self.nas_client.SQLITE_DB) as con: cur = con.cursor() cur.execute('INSERT INTO sync_queue VALUES(?,?,?,0,0,1)',[zvol, 'my_tank', 'compute-0-3']) con.commit() self.nas_client.zvol_unmapped( {'action': 'zvol_unmapped', 'target':target, 'zvol':zvol, 'status':'success'}, BasicProperties(reply_to='reply_to', correlation_id='message_id')) cur.execute('SELECT is_delete_remote FROM sync_queue WHERE zvol = ?',[zvol]) self.assertEqual(cur.fetchone()[0], 1) sync_result = MagicMock() sync_result.ready = MagicMock(return_value=True) self.nas_client.results[zvol] = sync_result self.nas_client.schedule_next_sync() self.assertFalse(self.check_zvol_busy(zvol)) print mock_run_command.mock_calls mock_run_command.assert_any_call(['su', 'img-storage', '-c', '/usr/bin/ssh compute-0-3 "/sbin/zfs destroy my_tank/%s -r"'%zvol])
def test_blocking_send_get(): connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time() channel.basic_publish(routing_key=queue_name, exchange="", body=message, properties=BasicProperties(content_type="text/plain", delivery_mode=1)) # Loop while we try to get the message we sent message_in = channel.basic_get(queue=queue_name) # Close the connection connection.close() # Only check the body if message_in[2] != message: assert False, "Did not receive the same message back"
def send_request(channel, id=None, retries = 3, timeout=1.0): id = id or str(uuid4()) # Our queue belongs to this process - we don't need it after the connection closes responses = channel.queue_declare(exclusive=True).method.queue try: consume = channel.consume(responses, inactivity_timeout=timeout, no_ack=True) try: # FIXME: this loop does not deal with a dropped connection for attempt in range(retries): channel.basic_publish( exchange='', routing_key='service', body='Request {}'.format(id), properties=BasicProperties( reply_to=responses, correlation_id=id)) response = next(consume) if not response: print "WARN: timeout" else: method, properties, body = response if id == properties.correlation_id: print "{}: {}".format(id, body) return True else: print "ERROR {}: unrecognised correlation_id {}".format(id, properties.correlation_id) print "ERROR: failed after {} retries".format(retries) return False finally: channel.cancel() finally: channel.queue_delete(responses)
def parse_server(instance, exchange, routing_key, body, properties=None, *_args, **_kwargs): """ 获取publish的一些必要信息,优先获取exchange作为metric名字 :return: """ global publish_headers host, port, byte, name = "Unknown", 0, 0, "Unknown" try: # blocking 和 其他异步生产者, connection = getattr(instance, 'connection', None) or getattr(instance, '_connection', None) impl = getattr(connection, '_impl', None) or connection host = impl.params.host port = impl.params.port publish_headers = getattr(properties, "headers", {}) or {} ty_headers, _external_id = process_cross_trace(None, msg_type='TingyunID') if ty_headers: from pika.spec import BasicProperties if not properties: properties = BasicProperties(headers=ty_headers) else: if not getattr(properties, "headers", None): setattr(properties, "headers", ty_headers) elif isinstance(properties.headers, dict): properties.headers.update(ty_headers) except Exception as err: console.info("Parse RabbitMQ host & port with error %s", err) _external_id = None _server = (host, port, sys.getsizeof(body), exchange or routing_key, _external_id) return _server, exchange, routing_key, body, properties, _args, _kwargs
def decorated_function( exchange: str, routing_key: str, body: bytes, properties: BasicProperties = None, mandatory: bool = False, ) -> Any: if not properties: properties = BasicProperties(headers={}) ctx = context.get_current() span = _get_span( tracer, channel, properties, span_kind=SpanKind.PRODUCER, task_name="(temporary)", ctx=ctx, operation=None, ) if not span: return original_function( exchange, routing_key, body, properties, mandatory ) with trace.use_span(span, end_on_exit=True): if span.is_recording(): propagate.inject(properties.headers) retval = original_function( exchange, routing_key, body, properties, mandatory ) return retval
def test_map_zvol_createnew_success(self, mockGetHostCommand, mockRunCommand): zvol = 'vol3' def my_side_effect(*args, **kwargs): if args[0][0] == 'zfs': return StringIO("") elif args[0][0] == 'tgt-setup-lun': return StringIO(tgt_setup_lun_response % (zvol, zvol)) mockRunCommand.side_effect = my_side_effect self.client.ib_net = 'ibnet' self.client.map_zvol( { 'action': 'map_zvol', 'zpool': 'mytank', 'zvol': zvol, 'remotehost': 'compute-0-1', 'size': '10' }, BasicProperties(reply_to='reply_to')) self.client.queue_connector.publish_message.assert_called_with( { 'action': 'map_zvol', 'zvol': zvol, 'nas': '%s.ibnet' % self.client.NODE_NAME, 'target': 'iqn.2001-04.com.nas-0-1-%s' % (zvol), 'size': '10' }, 'compute-0-1', self.client.NODE_NAME, on_fail=ANY) self.assertTrue(self.check_zvol_busy(zvol))
def task_process(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes) -> None: """ Process the received task :param channel: channel :param method: method :param properties: task properties :param body: task body :return: None """ raw_body = loads(body.decode(encoding="utf-8")) cases = raw_body.get("cases", {}) task = TaskItem(**raw_body.get("task", {})) try: results = list(self.manager.multi_case_runner(cases=cases)) for result in results: TaskCrud.create_task_result(task, result or {}) task.set_success( msg=f"Task done: {len(results)} out of {len(cases)} cases") except Exception as cases_err: task.set_error(msg=f"Task error: {str(cases_err)}") TaskCrud.update_task(task) logger.info(msg=f"Done task {task.task_id}") channel.basic_publish( exchange="", routing_key=properties.reply_to, properties=BasicProperties( correlation_id=properties.correlation_id), body=dumps(task.as_json()).encode(encoding="utf-8"), ) channel.basic_ack(delivery_tag=method.delivery_tag)
def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: exchange = config.get("exchange") routing_key = config["routing_key"] connection = create_connection(config=config) channel = connection.channel() streams = {s.stream.name for s in configured_catalog.streams} try: for message in input_messages: if message.type == Type.STATE: # Emitting a state message means all records that came before it # have already been published. yield message elif message.type == Type.RECORD: record = message.record if record.stream not in streams: # Message contains record from a stream that is not in the catalog. Skip it! continue headers = {"stream": record.stream, "emitted_at": record.emitted_at, "namespace": record.namespace} properties = BasicProperties(content_type="application/json", headers=headers) channel.basic_publish( exchange=exchange or "", routing_key=routing_key, properties=properties, body=json.dumps(record.data) ) else: # Let's ignore other message types for now continue finally: connection.close()
def test_map_zvol_createnew_missing_blkdev_error(self, mockSyncEnabled, mockRunCommand): zvol = 'vol2' target = 'iqn.2001-04.com.nas-0-1-%s' % zvol bdev = 'sdc' mockRunCommand.side_effect = self.create_iscsiadm_side_effect( target + "_missing_target", bdev) self.client.map_zvol( { 'action': 'map_zvol', 'target': target, 'nas': 'nas-0-1', 'size': '35', 'zvol': zvol }, BasicProperties(reply_to='reply_to', message_id='message_id')) self.client.queue_connector.publish_message.assert_called_with( { 'action': 'zvol_mapped', 'status': 'error', 'target': target, 'error': 'Not found %s in targets' % target }, 'reply_to', reply_to=self.client.NODE_NAME, correlation_id='message_id')
def test_map_zvol_busy(self, mockGetHostCommand, mockRunCommand): zvol = 'vol3_busy' def my_side_effect( *args, **kwargs): # just in case... not used in normal condition if args[0][0] == 'zfs': return StringIO("") elif args[0][0] == 'tgt-setup-lun': return StringIO(tgt_setup_lun_response % (zvol, zvol)) mockRunCommand.side_effect = my_side_effect self.client.ib_net = 'ibnet' self.client.map_zvol( { 'action': 'map_zvol', 'zpool': 'mytank', 'zvol': zvol, 'remotehost': 'compute-0-1', 'size': '10' }, BasicProperties(reply_to='reply_to')) self.client.queue_connector.publish_message.assert_called_with( { 'action': 'zvol_mapped', 'status': 'error', 'error': 'ZVol %s is busy' % zvol }, routing_key='reply_to', exchange='') self.assertTrue(self.check_zvol_busy(zvol))
def test_map_zvol_unmap_error(self, mockRunCommand): zvol = 'vol2' target = 'iqn.2001-04.com.nas-0-1-%s' % zvol bdev = 'sdc' def my_side_effect(*args, **kwargs): if args[0][:3] == ['iscsiadm', '-m', 'session']: return StringIO(iscsiadm_session_response % (target, bdev)) elif args[0][:3] == ['iscsiadm', '-m', 'discovery']: return StringIO(iscsiadm_discovery_response % target) # find remote targets elif args[0][:3] == ['iscsiadm', '-m', 'node']: raise ActionError('Some error happened') mockRunCommand.side_effect = my_side_effect self.client.unmap_zvol( { 'action': 'unmap_zvol', 'target': target, 'zvol': zvol }, BasicProperties(reply_to='reply_to', message_id='message_id')) self.client.queue_connector.publish_message.assert_called_with( { 'action': 'zvol_unmapped', 'status': 'error', 'target': target, 'zvol': zvol, 'error': 'Some error happened' }, 'reply_to', reply_to=self.client.NODE_NAME, correlation_id='message_id') mockRunCommand.assert_called_with( ['iscsiadm', '-m', 'node', '-T', target, '-u'])
def test_map_zvol_createnew_success(self, mockSyncEnabled, mockRunCommand): zvol = 'vol2' target = 'iqn.2001-04.com.nas-0-1-%s' % zvol bdev = 'sdc' mockRunCommand.side_effect = self.create_iscsiadm_side_effect( target, bdev) self.client.map_zvol( { 'action': 'map_zvol', 'target': target, 'nas': 'nas-0-1', 'size': '35', 'zvol': zvol }, BasicProperties(reply_to='reply_to', message_id='message_id')) self.client.queue_connector.publish_message.assert_called_with( { 'action': 'zvol_mapped', 'status': 'success', 'bdev': '/dev/%s' % bdev, 'target': target }, 'reply_to', reply_to=self.client.NODE_NAME, correlation_id='message_id') mockRunCommand.assert_any_call([ 'iscsiadm', '-m', 'discovery', '-t', 'sendtargets', '-p', 'nas-0-1' ]) mockRunCommand.assert_any_call( ['iscsiadm', '-m', 'node', '-T', target, '-p', 'nas-0-1', '-l']) mockRunCommand.assert_any_call(['iscsiadm', '-m', 'session', '-P3']) assert 3 == mockRunCommand.call_count
def send_request(channel, timeout=1.0, id=None): id = id or str(uuid4()) channel.confirm_delivery() # We use a durable queue named after the request. It's only allowable to call this # function again with the same id. responses = channel.queue_declare(queue='service-response-{}'.format(id), durable=True).method.queue consume = channel.consume(responses, inactivity_timeout=timeout, exclusive=True) try: if channel.basic_publish( exchange='', routing_key='service', body='Request {}'.format(id), properties=BasicProperties( reply_to=responses, delivery_mode=2)): for response in consume: method, properties, body = response print body channel.basic_ack(method.delivery_tag) channel.queue_delete(responses) return True return False else: return False finally: channel.cancel()
def test_blocking_invalid_exchange(): # Connect to RabbitMQ connection = BlockingConnection(support.PARAMETERS) # Open the channel channel = connection.channel() # Declare the queue queue_name = support.tools.test_queue_name('blocking_send_get') channel.queue_declare(queue=queue_name, durable=False, exclusive=True, auto_delete=True) message = 'test_blocking_send:%.4f' % time() try: channel.basic_publish(exchange="invalid-exchange", routing_key=queue_name, body=message, mandatory=True, properties=BasicProperties( content_type="text/plain", delivery_mode=1)) while True: channel.transport.connection.process_data_events() except AMQPChannelError, err: if err[0] != 404: assert False, "Did not receive a Channel.Close"
def configure(self): self.consumer_tag = 'R2' self.delivery_tag = '1138' self.exchange = 'droids' self.redelivered = True self.routing_key = 'dejarik' self.app_id = '3PO' self.content_encoding = 'GZIP' self.content_type = 'application/JSON' self.correlation_id = 'dockingbay94' self.expiration = '60000' # 1 minute TTL self.headers = {'tags': ['best practices']} self.message_id = '1234' self.priority = 9 self.reply_to = 'reply_to_address' self.timestamp = '1234567890' self.type = 'message_type_name' self.user_id = 'amqp_user_id' self.channel = pika.channel.Channel(pika.connection.Connection(), channel_number=1) self.method = Basic.Deliver(**dict([(x, getattr(self, x)) for x in self.method_delegates])) self.body = '{"strategy": "Let the Wookie win."}' self.header = BasicProperties( **dict([(x, getattr(self, x)) for x in self.properties_delegates]))
def test_unmap_zvol_not_found(self, mockRunCommand): zvol = 'vol2' target = 'iqn.2001-04.com.nas-0-1-%s' % zvol bdev = 'sdc' mockRunCommand.side_effect = self.create_iscsiadm_side_effect( target, bdev) self.client.unmap_zvol( { 'action': 'unmap_zvol', 'target': target + "not_found", 'zvol': zvol }, BasicProperties(reply_to='reply_to', message_id='message_id')) print self.client.queue_connector.publish_message.mock_calls self.client.queue_connector.publish_message.assert_called_with( { 'action': 'zvol_unmapped', 'status': 'success', 'target': target + "not_found", 'zvol': zvol }, 'reply_to', reply_to=self.client.NODE_NAME, correlation_id='message_id')
def publish_properties(self, channel=None): if channel and '!' in channel: headers = {'asgi_channel': channel.rsplit('!')[-1]} else: headers = None expiration = str(self.expiry * 1000) properties = BasicProperties(headers=headers, expiration=expiration) return properties
def test_teardown_busy(self, mockHostSyncAttr, mockRunCommand): zvol = 'vol3_busy' mockRunCommand.return_value = StringIO(tgtadm_response % (zvol, zvol)) self.assertFalse( self.client.unmap_zvol({ 'action': 'unmap_zvol', 'zvol': zvol }, BasicProperties(reply_to='reply_to')))
def prepare_message(self, message_data, priority=None, content_type=None, content_encoding=None, headers=None, properties=None): properties = BasicProperties(priority=priority, content_type=content_type, content_encoding=content_encoding, headers=headers, **properties) return message_data, properties
def _amqp_basic_properties(self, content_type, content_encoding, delivery_mode, priority, expiration, user_id, app_id): return BasicProperties(content_type=content_type, content_encoding=content_encoding, delivery_mode=delivery_mode, priority=priority, expiration=expiration, user_id=user_id, app_id=app_id)
def publish(self, exchange, routing_key, payload, properties=None): """Publish a message to RabbitMQ on the same channel the original message was received on, automatically serializing the message payload. :param str exchange: the exchange to publish to :param str routing_key: the routing key to publish with :param pikachewie.data.Properties: the message properties :param dict|list: the message body to publish """ if not properties: properties = BasicProperties() properties.content_type = 'application/json' super(JSONPublisherMixin, self).publish( exchange, routing_key, self._serialize(payload), properties, )
def test_wrapping_function(subscriber): mock_fn = mock.MagicMock() subscriber.bind('some.event')(mock_fn) bound_fn = subscriber._late_bindings[('test_exchange', '')][0].callback props = BasicProperties(type='some.event') bound_fn(None, None, props, json.dumps({'hello': 'world'})) mock_fn.assert_called_once_with('some.event', {'hello': 'world'}, props)
def publish(self, dict): global rpc_subscriber while rpc_subscriber.consumer_tag is None: continue props = BasicProperties() props.reply_to = rpc_subscriber.queue_name props.correlation_id = str(uuid.uuid4()) try: rpc_subscriber.rpc_data[props.correlation_id] = Event() self.publisher.publish(dict, props) except: self.logger.exception("Error publishing rpc message") del rpc_subscriber.rpc_data[props.correlation_id] props.correlation_id = None self.publisher.close() return props.correlation_id
def publish(self, exchange, routing_key, body, durable=False): if durable: properties = BasicProperties( delivery_mode=2) # make message persistent else: properties = None self.channel.basic_publish(exchange=exchange, routing_key=routing_key, body=body, properties=properties)
def test_envelope(self): prop = BasicProperties(content_type='application/json', content_encoding='utf-8', headers=None, delivery_mode=2, priority=7, correlation_id='some-correlation-id', reply_to='reply_to_name', expiration='some-expiration', message_id='some-message-id', timestamp='123456789', type='message-type-a', user_id='user-id', app_id='app-id', cluster_id='cluster-id') payload = 'some data' delivery_info = Basic.Deliver(consumer_tag=34, delivery_tag=132, redelivered=True, exchange='exchange1', routing_key='routing_key1') envelope = Envelope(prop, payload, delivery_info) self.assertEqual(envelope.payload, payload) self.assertEqual(envelope.properties, prop) self.assertEqual(envelope.delivery_info, delivery_info) self.assertEqual(envelope.consumer_tag, delivery_info.consumer_tag) self.assertEqual(envelope.delivery_tag, delivery_info.delivery_tag) self.assertEqual(envelope.redelivered, delivery_info.redelivered) self.assertEqual(envelope.routing_key, delivery_info.routing_key) self.assertEqual(envelope.exchange, delivery_info.exchange) self.assertEqual(envelope.content_type, prop.content_type) self.assertEqual(envelope.content_encoding, prop.content_encoding) self.assertEqual(envelope.headers, prop.headers) self.assertEqual(envelope.delivery_mode, prop.delivery_mode) self.assertEqual(envelope.priority, prop.priority) self.assertEqual(envelope.correlation_id, prop.correlation_id) self.assertEqual(envelope.reply_to, prop.reply_to) self.assertEqual(envelope.expiration, prop.expiration) self.assertEqual(envelope.message_id, prop.message_id) self.assertEqual(envelope.timestamp, prop.timestamp) self.assertEqual(envelope.type, prop.type) self.assertEqual(envelope.user_id, prop.user_id) self.assertEqual(envelope.app_id, prop.app_id) self.assertEqual(envelope.cluster_id, prop.cluster_id) self.assertEqual(envelope.headers, prop.headers) self.assertIsNone(envelope.get_header('some_header')) envelope.set_header('some_header', 50) envelope.set_header('another_header', 'hello') self.assertEqual(envelope.headers, dict(some_header=50, another_header='hello')) self.assertEqual(envelope.get_header('some_header'), 50) self.assertEqual(envelope.get_header('another_header'), 'hello') self.assertIsNone(envelope.get_header('non_exist_header'))
def publish_message(self, **kwargs): if self._channel: try: pro = BasicProperties(content_type='application/json', delivery_mode=2) self._channel.basic_publish(exchange=self.EXCHANGENAME, routing_key=self.ROUTINGKEY, body=json.dumps(kwargs), properties=pro) print("message has sent --> {0} --> {1}".format(kwargs, time.time())) except BaseException as e: print(e) else: print("does not exist channel")
def publish_properties(self, channel=None): """AMQP message properties.""" # Store local part of the process local channel in the AMQP # message header. if channel and '!' in channel: headers = {'asgi_channel': channel.rsplit('!')[-1]} else: headers = None # Specify message expiration in milliseconds. expiration = str(self.expiry * 1000) properties = BasicProperties(headers=headers, expiration=expiration) return properties
def publish(self, dict): props = BasicProperties() props.correlation_id = self.correlation_id if self.channel.is_open: self.channel.basic_publish(exchange='', routing_key=self.queue_name, properties=props, body=json.dumps(dict))