def connection_thread(url, results, hide_password=False): from oslo_config import cfg from oslo_messaging.transport import TransportURL from pika import exceptions as pika_exceptions from pika import URLParameters as PikaUrlParameters from pika import BlockingConnection as PikaBlockingConnection try: parsed_url = TransportURL.parse(cfg.CONF, url) if hide_password: url = re.sub(':+[^:@]+@', ':******@', url) except Exception as e: results.append({'url': url, 'exception': e}) else: test_url, driver = parse_test_url(parsed_url) try: if driver == 'kombu': connection = Connection(test_url) connection.connect() connection.close() elif driver == 'pika': params = PikaUrlParameters(test_url) params.socket_timeout = 5 conn = PikaBlockingConnection(params) conn.close() except (OSError, pika_exceptions.ConnectionClosed): results.append({'url': url, 'exception': _('Url not reachable')}) except (AccessRefused, pika_exceptions.ProbableAuthenticationError): results.append({ 'url': url, 'exception': _('Credentials incorrect') }) except Exception as e: results.append({'url': url, 'exception': force_text(e)}) else: results.append({'url': url})
def _connect(self): try: logger.info("attempt to open connection", server="primary", category="rabbitmq") return BlockingConnection(URLParameters(self.rabbitmq_url)) except AMQPError as e: logger.error( "unable to open connection", exc_info=e, server="primary", category="rabbitmq", ) try: logger.info( "attempt to open connection", server="secondary", category="rabbitmq", ) return BlockingConnection( URLParameters(self.rabbitmq_secondary_url)) except AMQPError as err: logger.error( "unable to open connection", exc_info=e, server="secondary", category="rabbitmq", ) raise err
def start(self): """Start worker.""" connected = False # Define connection while (not connected): try: self._creds = self._amqp_url.split('amqp://')[1].split("@")[0] parameters = URLParameters(self._amqp_url) self._host = parameters.host self._port = "15672" self._vhost = parameters.virtual_host self._blocking_connection = BlockingConnection(parameters) self._connection = SelectConnection( URLParameters(self._amqp_url), on_open_callback=self.on_open_connection) self._connection.ioloop.start() connected = True # Catch a Keyboard Interrupt to make sure that the connection is closed cleanly except KeyboardInterrupt: # Gracefully close the connection self._connection.close() # Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed self._connection.ioloop.start() except: if (self._debug): print( " [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..." .format(self._reconection_time)) time.sleep(self._reconection_time)
def create_connection(self): url_params = self._config.MQ_URI future = Future() TornadoConnection(URLParameters(url_params), partial(self.on_connection_open, future), partial(self.on_open_error, future)) return future
def decrypt_and_write(): with open(settings.SDX_SEFT_CONSUMER_KEYS_FILE) as file: keys = yaml.safe_load(file) key_store = KeyStore(keys) connection = BlockingConnection(URLParameters(settings.RABBIT_URL)) channel = connection.channel() method, properties, body = channel.basic_get( settings.RABBIT_QUARANTINE_QUEUE) if method: logger.info("Recovered quarantine message", body=body, headers=properties.headers) try: decrypted_message = decrypt(body.decode("utf-8"), key_store, KEY_PURPOSE_CONSUMER) payload = SeftConsumer.extract_file(decrypted_message, properties.headers['tx_id']) with open('/tmp/{}'.format(payload.file_name), 'wb') as recovered_file: recovered_file.write(payload.decoded_contents) channel.basic_ack(method.delivery_tag) logger.info("Message ACK") except (InvalidTokenException, ValueError): logger.exception("Bad decrypt") channel.basic_nack(method.delivery_tag) logger.info("Nacking message") except Exception: logger.exception("Failed to process") channel.basic_nack(method.delivery_tag) logger.info("Nacking message") else: logger.info('No message found on quarantine queue')
def __init__(self, rabbitmq_url, io_loop=None): """ A Rabbitmq client, using tornado to complete asynchronous invoking. It is an `everything-in-one` RabbitMQ client, including following interfaces: - establish connection - publish - receive - rpc details: 1. establish connection Every instance of `TornadoAdapter` has two rabbitmq connections. one for publish and the other for consumer. 2. publish Publishing connection creates a channel when invoke publish method. After publishing message, it closes channel. 3. receive It receives message from queue and process message. and publish message if necessary. 4. rpc It invoke rpc call and wait result asynchronously. Of course, it supports timeout. :param rabbitmq_url: url for rabbitmq. it can be either '127.0.0.1' ("localhost") or 'amqp://*****:*****@10.12.7.22:5672/' :param io_loop: io loop. if it is none, using IOLoop.current() instead. """ self._parameter = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \ URLParameters(rabbitmq_url) if io_loop is None: io_loop = IOLoop.current() self._io_loop = io_loop self._logger = logging.getLogger(__name__) self._publish_conn = _AsyncConnection(rabbitmq_url, io_loop) self._receive_conn = _AsyncConnection(rabbitmq_url, io_loop) self._rpc_exchange_dict = dict() self._rpc_corr_id_dict = dict()
def reprocess(): # To use this script, fill in the credentials for rabbit. They should be found in app/settings.py rabbit_url = 'amqp://<user>:<pass>@<host>:<port>/%2f' connection = BlockingConnection(URLParameters(rabbit_url)) channel = connection.channel() method, properties, body = channel.basic_get('Seft.Responses.Quarantine') if method: try: print("Recovered quarantine message") print("Headers:") print(properties.headers) # Uncomment if extra information is needed (payload is encrypted so it's not likely to be useful) # print("Body:") # print(body) publisher = QueuePublisher(urls=[rabbit_url], queue='Seft.Responses') publisher.publish_message(body, headers=properties.headers) print("Message successfully reprocessed") channel.basic_ack(method.delivery_tag) print("Message ACK") except PublishMessageError as e: print(e) channel.basic_nack(method.delivery_tag) else: print('No message found on quarantine queue')
def __init__(self, connection_string: str, queue: str, tasks=None, loglevel="WARNING"): self._queue = queue self._tasks = tasks or {} self._logger = logging.getLogger(queue) stream_handler = logging.StreamHandler() # TODO: сделать кастомнй форматер чтобы добавлять пробелы (https://stackoverflow.com/questions/6692248/python-logging-string-formatting/22707429) stream_format = logging.Formatter( '[NokiMQ] - %(asctime)s [%(name)s] %(levelname)s %(message)s' ) stream_handler.setFormatter(stream_format) self._logger.addHandler(stream_handler) self._logger.setLevel(logging.getLevelName(loglevel)) self._logger.info(f'Connecting to "{connection_string}"') connection = None try: connection = BlockingConnection(URLParameters(connection_string)) except: self._logger.critical('Connection failed') quit(-1) self._logger.info('Connected!') self._channel = connection.channel() self._channel.queue_declare(queue)
def _connect(self): try: self.connection = BlockingConnection( URLParameters(settings.EQ_RABBITMQ_URL)) except AMQPError as e: logger.error("unable to open rabbit mq connection", exc_info=e, rabbit_url=settings.EQ_RABBITMQ_URL) try: self.connection = BlockingConnection( URLParameters(settings.EQ_RABBITMQ_URL_SECONDARY)) except AMQPError as err: logger.error("unable to open rabbit mq connection", exc_info=e, rabbit_url=settings.EQ_RABBITMQ_URL_SECONDARY) raise err
def __init__(self, amqp_url=None, queue_name=None, on_message_callback=None, panic_event=None, logger=None, thread_name=None, **kwargs): self._connection = None self._channel = None self._consumer_tag = None self._queue_name = queue_name self._on_message_callback = on_message_callback self._panic_event = panic_event self._max_concurrent = kwargs.get("max_concurrent", 1) self.logger = logger or logging.getLogger(__name__) if "connection_info" in kwargs: params = kwargs["connection_info"] # Default to one attempt as the Plugin implements its own retry logic params["connection_attempts"] = params.get("connection_attempts", 1) self._connection_parameters = PikaClient( **params).connection_parameters() else: self._connection_parameters = URLParameters(amqp_url) super(RequestConsumer, self).__init__(name=thread_name)
def __init__(self, rabbitmq_url, io_loop, timeout=10): self._parameters = ConnectionParameters("127.0.0.1") if rabbitmq_url in ["localhost", "127.0.0.1"] else \ URLParameters(rabbitmq_url) self._io_loop = io_loop self._timeout = timeout self._connection_queue = Queue(maxsize=1) self.current_status = self.INIT_STATUS
def __init__(self, rabbitmq_url, configuration: Config, io_loop=None): super().__init__() self._parameter = ConnectionParameters("127.0.0.1") \ if rabbitmq_url in ["localhost", "127.0.0.1"] else URLParameters(rabbitmq_url) if io_loop is None: io_loop = IOLoop.current() self.io_loop = io_loop self.configuration = configuration self.pub_connection = AsyncConnection(rabbitmq_url, io_loop) configpub = configuration["publish"].values() self.pub_channels = self.create_channel('exchange', self.pub_connection, *configpub) self.rcv_connection = AsyncConnection(rabbitmq_url, io_loop) configrcv = configuration["receive"].values() self.rcv_channels = self.create_channel('queue', self.rcv_connection, *configrcv) # print(self.rcv_channels) self._rpc_corr_id_dict = dict()
def _run_amqp(self): url = self.output_id.amqp_host_id.connection connection = BlockingConnection(URLParameters(url)) channel = connection.channel() result = channel.basic_publish(**self._generate_amqp_data()) _logger.debug(result) connection.close()
def amqp() -> BlockingConnection: try: dsn: str = Config.amqp_url() return BlockingConnection(parameters=URLParameters(dsn)) except AMQPConnectionError: logging.error("Couldn't connect to the AMQP broker at: [%s]" % dsn) sys.exit(2)
def __init__(self, ampq_url=None): if not ampq_url: ampq_url = self._build_ampq_url() parameters = URLParameters(ampq_url) self.channel = BlockingConnection(parameters).channel() self.channel.confirm_delivery() self.process_functions = {}
def connect(self): """ Creating connection object """ url = self.get_url() logger.info('Connecting to %s', url) return SelectConnection( URLParameters(url), self.on_connection_open, stop_ioloop_on_close=False )
def __init__(self, url, listened_queue, request_exchange, response_exchange, response_routing_key): super(EchoServer, self).__init__() self.connection_parameters = URLParameters(url) self.listened_queue = listened_queue self.request_exchange = request_exchange self.response_exchange = response_exchange self.response_routing_key = response_routing_key self.channel = None self.connection = None self.event = Event()
def get_connection(): while True: try: connc = BlockingConnection( URLParameters( os.getenv("AMQP_URL", "amqp://*****:*****@localhost:5672/"))) if connc.is_open: break except Exception: pass return connc
def test_new_pika_invalid_ca_cert(self): """Assert a ConfigurationException is raised when the CA can't be opened.""" params = URLParameters('amqps://myhost') tls_conf = { 'keyfile': os.path.join(FIXTURES_DIR, 'key.pem'), 'certfile': os.path.join(FIXTURES_DIR, 'cert.pem'), 'ca_cert': os.path.join(FIXTURES_DIR, 'invalid_ca.pem'), } with mock.patch.dict(config.conf, {'tls': tls_conf}): self.assertRaises(ConfigurationException, _session._configure_tls_parameters, params) self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def __init__(self, url: str, exchange: str, routing_key: str): try: self.conn = BlockingConnection(parameters=URLParameters(url)) self.channel = self.conn.channel() logging.info(f'Successfully connected to AMQP Broker at [{url}]') self.exchnage = exchange self.routing_key = routing_key # Declare the queue self.channel.queue_declare(routing_key, durable=True) self.channel.confirm_delivery() except AMQPConnectionError: logging.error(f'Couldn\'t connect to the AMQP broker at [{url}]') sys.exit(2)
def test_new_pika(self): """Assert configuring a cert and key results in a TLS connection with new pika versions.""" params = URLParameters("amqps://myhost") tls_conf = { "keyfile": os.path.join(FIXTURES_DIR, "key.pem"), "certfile": os.path.join(FIXTURES_DIR, "cert.pem"), "ca_cert": os.path.join(FIXTURES_DIR, "ca_bundle.pem"), } with mock.patch.dict(config.conf, {"tls": tls_conf}): _session._configure_tls_parameters(params) self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def _connect(self): try: logger.info('attempt to open connection', server='primary', category='rabbitmq') return BlockingConnection(URLParameters(self.rabbitmq_url)) except AMQPError as e: logger.error('unable to open connection', exc_info=e, server='primary', category='rabbitmq') try: logger.info('attempt to open connection', server='secondary', category='rabbitmq') return BlockingConnection( URLParameters(self.rabbitmq_secondary_url)) except AMQPError as err: logger.error('unable to open connection', exc_info=e, server='secondary', category='rabbitmq') raise err
def test_new_pika_invalid_key(self): """Assert a ConfigurationException is raised when the key can't be opened.""" params = URLParameters("amqps://myhost") tls_conf = { "keyfile": os.path.join(FIXTURES_DIR, "invalid_key.pem"), "certfile": os.path.join(FIXTURES_DIR, "cert.pem"), "ca_cert": os.path.join(FIXTURES_DIR, "ca_bundle.pem"), } with mock.patch.dict(config.conf, {"tls": tls_conf}): self.assertRaises(ConfigurationException, _session._configure_tls_parameters, params) self.assertTrue(isinstance(params.ssl_options, _session.SSLOptions))
def connect(self): """This method connects to RabbitMQ, returning the connection handle. When the connection is established, the on_connection_open method will be invoked by pika. :rtype: pika.SelectConnection """ url = self.get_url() logger.info('Connecting to %s', url) return SelectConnection( parameters=URLParameters(url), on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed)
def __init__(self, data): self.corr_id = str(uuid4()) self.json_input = data self.response = None url = environ.get('CLOUDAMQP_URL', 'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum') params = URLParameters(url) self.connection = BlockingConnection(params) self.channel = self.connection.channel() result = self.channel.queue_declare(queue='', exclusive=True) self.callback_queue = result.method.queue self.channel.basic_consume( queue=self.callback_queue, on_message_callback=self.on_response, auto_ack=True)
def replica_channel(settings): if current_transport is not RabbitMQTransport: pytest.skip("Replica channel is implemented only for RabbitMQTransport.") connection = BlockingConnection( parameters=URLParameters(settings.CQRS['url']), ) rabbit_mq_channel = connection.channel() rabbit_mq_channel.queue_purge('replica') rabbit_mq_channel.queue_purge('dead_letter_replica') yield rabbit_mq_channel connection.close()
def test_old_pika_approach_no_cert(self): """Assert if no cert is provided, no key is passed to pika either.""" params = URLParameters("amqps://") tls_conf = {"keyfile": "key.pem", "certfile": None, "ca_cert": "ca_bundle.pem"} expected_options = { "keyfile": None, "certfile": None, "ca_certs": "ca_bundle.pem", "cert_reqs": ssl.CERT_REQUIRED, "ssl_version": ssl.PROTOCOL_TLSv1_2, } with mock.patch.dict(config.conf, {"tls": tls_conf}): _session._configure_tls_parameters(params) self.assertTrue(params.ssl) self.assertEqual(params.ssl_options, expected_options)
def connect(self) -> None: """ For now we use a synchronous connection - caller is blocked until a message is added to the queue. We might switch to asynch connections should this incur noticable latencies. """ params = URLParameters(self.connection_settings) connection = BlockingConnection(params) channel = connection.channel() # set up the Exchange (if it does not exist) channel.exchange_declare(exchange=self.exchange, exchange_type=self.exchange_type, durable=True, auto_delete=False) self.connection = connection self.channel = channel
def main(self): url = environ.get( 'CLOUDAMQP_URL', 'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum' ) params = URLParameters(url) connection = BlockingConnection(params) channel = connection.channel() print('start') query = Query() query.connect() while True: inmessage = channel.basic_get("received", auto_ack=True) sleep(0.2) if inmessage[2] is not None: print('ok') while True: self.data = query.select() sleep(0.8)
def connect(cls, url): f = Future() def on_open(conn): f.set_result(cls(conn)) def on_open_error(conn, err): f.set_exception(AMQPConnectionError(err)) def on_close(conn): LOGGER.debug('connection closed: %s', conn) TornadoConnection( URLParameters(url), on_open_callback=on_open, on_open_error_callback=on_open_error, on_close_callback=on_close, ) return f