def __init__(self, uuid, exchange_name, on_message, on_wait=None, **kwargs): self._uuid = uuid self._exchange_name = exchange_name self._on_message = on_message self._on_wait = on_wait self._running = threading.Event() self._url = kwargs.get('url') self._transport = kwargs.get('transport') self._transport_opts = kwargs.get('transport_options') # create connection self._conn = kombu.Connection(self._url, transport=self._transport, transport_options=self._transport_opts) # create exchange self._exchange = kombu.Exchange(name=self._exchange_name, channel=self._conn, durable=False, auto_delete=True)
def run(connection, args): exchange = kombu.Exchange(name='villas', type='headers') headers = SimulatorCommand.get_headers(args) headers['x-match'] = 'any' if len(headers) > 0 else 'all' queue = kombu.Queue(exchange=exchange, binding_arguments=headers, durable=False) consumer = kombu.Consumer(connection, queues=queue, on_message=MonitorCommand.on_message) try: with consumer: while True: connection.drain_events() except KeyboardInterrupt: pass
def start(self): self._browsing_threads = set() self._browsing_threads_lock = threading.Lock() self._exchange = kombu.Exchange(name=self.exchange_name, type='direct', durable=True) self._reconnect_requested = False self._producer = None self._producer_lock = threading.Lock() with self._producer_lock: self._producer_conn = kombu.Connection(self.amqp_url) self._producer = self._producer_conn.Producer(serializer='json') self._consumer_thread = threading.Thread(target=self._consume_amqp, name='AmqpConsumerThread') self._consumer_stop = threading.Event() self._consumer_thread.start()
def __init__(self, url=None, prefix='asgi:', expiry=60, group_expiry=86400, capacity=100, channel_capacity=None): super(AMQPChannelLayer, self).__init__( expiry=expiry, group_expiry=group_expiry, capacity=capacity, channel_capacity=channel_capacity, ) kombu.serialization.enable_insecure_serializers() self.url = url or 'amqp://*****:*****@localhost:5672/%2F' self.prefix = prefix + 'tower:{}'.format(socket.gethostname()) self.exchange = kombu.Exchange(self.prefix, type='topic') self.tdata = threading.local()
def notify_radius(state, priority) -> int: config = get_config(runtime_checks=True) queue_name = config.HADES_CELERY_NODE_QUEUE exchange_name = config.HADES_CELERY_RPC_EXCHANGE exchange_type = config.HADES_CELERY_RPC_EXCHANGE_TYPE routing_key = config.HADES_CELERY_SITE_ROUTING_KEY exchange = kombu.Exchange(exchange_name, exchange_type) with closing(app.connection(connect_timeout=1)) as connection: queue = app.amqp.queues[queue_name] bound_queue = queue.bind(connection.default_channel) if state == 'MASTER': logger.info("Binding site node queue %s to RPC exchange %s " "with site routing key %s", queue_name, exchange_name, routing_key) bound_queue.bind_to(exchange=exchange, routing_key=routing_key) else: logger.info("Unbinding site node queue %s from RPC exchange %s " "with site routing key %s", queue_name, exchange_name, routing_key) bound_queue.unbind_from(exchange=exchange, routing_key=routing_key) return 0
def _hub_bootstrap(self): with connections[self.conn].acquire(block=True) as conn: exchange = kombu.Exchange( name=self.exchange, type=self.transport, durable=True, channel=conn, ) exchange.declare() if self.transport == ETransport.DIRECT: q = 'sameQ' queue = kombu.Queue(name=q, exchange=exchange, routing_key=self.rkey, channel=conn, message_ttl=600, queue_arguments={'x-queue-type': 'classic'}, durable=True) queue.declare() # It will run forever to consume msg. C(conn, [queue], self._handle).run()
def pulse_consumer(connection, exchange, request): exchange_name = 'exchange/treeherder/v1/{}'.format(exchange) exchange = kombu.Exchange(name=exchange_name, type='topic') queue = kombu.Queue( no_ack=True, exchange=exchange, # Exchange name routing_key='#', # Bind to all messages auto_delete=True, # Delete after each test exclusive=False) # Disallow multiple consumers simpleQueue = connection.SimpleQueue(name=queue, channel=connection, no_ack=True) def fin(): connection.release() request.addfinalizer(fin) return simpleQueue
def run(connection, args): channel = connection.channel() exchange = kombu.Exchange('villas', type='headers', durable=True) producer = kombu.Producer(channel, exchange=exchange) consumer = kombu.Consumer(channel, queues=kombu.Queue(exchange=exchange, durable=False), on_message=SimulatorPingCommand.on_message) message = {'action': 'ping'} producer.publish(message, headers=SimulatorCommand.get_headers(args)) with consumer: try: while True: connection.drain_events(timeout=10) except socket.timeout: pass
def publish(self, exchange_name, routing_key, payload): with self.connection as connection: if not connection.connected: connection.connect() exchange = kombu.Exchange(exchange_name, type="topic") message = { "payload": payload, "_meta": { "exchange": exchange_name, "routing_key": routing_key, "serializer": "json", "sent": datetime.datetime.utcnow().isoformat() }, } producer = connection.Producer(exchange=exchange, routing_key=routing_key, serializer="json") producer.publish(message) connection.close()
def _setup_connection(self): """Returns True if a valid connection exists already, or if one can be created.""" if self.conn: return True id_conf = read_conf(ID_CONF_FILE_NAME) # The indentity.yaml file contains either a singular string variable # 'rabbit_host', or a comma separated list in the plural variable # 'rabbit_hosts' host = None hosts = id_conf.get('rabbit_hosts', None) if hosts is not None: host = hosts.split(",")[0] else: host = id_conf.get('rabbit_host', None) if host is None: log.warning("no host info in configuration, can't set up rabbit.") return False try: url = "amqp://{}:{}@{}/{}".format(id_conf['rabbit_userid'], id_conf['rabbit_password'], host, id_conf['rabbit_virtual_host']) self.conn = kombu.BrokerConnection(url) self.exchange = kombu.Exchange("glance-simplestreams-sync-status") status_queue = kombu.Queue("glance-simplestreams-sync-status", exchange=self.exchange) status_queue(self.conn.channel()).declare() except: log.exception("Exception during kombu setup") return False return True
def __init__(self, rabbit_ip, rabbit_port, rabbit_user, rabbit_password, rabbit_vhost, rabbit_ha_mode, q_name, subscribe_cb, logger, **kwargs): self._rabbit_ip = rabbit_ip self._rabbit_port = rabbit_port self._rabbit_user = rabbit_user self._rabbit_password = rabbit_password self._rabbit_vhost = rabbit_vhost self._subscribe_cb = subscribe_cb self._logger = logger self._publish_queue = Queue() self._conn_lock = Semaphore() self.obj_upd_exchange = kombu.Exchange('vnc_config.object-update', 'fanout', durable=False) self._ssl_params = self._fetch_ssl_params(**kwargs) # Register a handler for SIGTERM so that we can release the lock # Without it, it can take several minutes before new master is elected # If any app using this wants to register their own sigterm handler, # then we will have to modify this function to perhaps take an argument gevent.signal(signal.SIGTERM, self.sigterm_handler)
def publish(self, exchange_name, routing_key, payload): with self.connection as connection: if not connection.connected: connection.connect() exchange = kombu.Exchange(exchange_name, type='topic') message = { 'payload': payload, '_meta': { 'exchange': exchange_name, 'routing_key': routing_key, 'serializer': 'json', 'sent': datetime.datetime.utcnow().isoformat()}, } producer = connection.Producer( exchange=exchange, routing_key=routing_key, serializer='json', ) producer.publish(message) connection.close()
def _configureKombu(self): """ Configure kombu for rabbitmq """ try: connString = 'amqp://{0}:{1}@{2}:{3}//'.format( RABBITMQ['mquser'], RABBITMQ['mqpassword'], RABBITMQ['mqserver'], RABBITMQ['mqport']) self.mqConn = kombu.Connection(connString) self.alertExchange = kombu.Exchange(name=RABBITMQ['alertexchange'], type='topic', durable=True) self.alertExchange(self.mqConn).declare() alertQueue = kombu.Queue(RABBITMQ['alertqueue'], exchange=self.alertExchange) alertQueue(self.mqConn).declare() self.mqproducer = self.mqConn.Producer(serializer='json') self.log.debug('Kombu configured') except Exception as e: self.log.error( 'Exception while configuring kombu for alerts: {0}'.format(e))
def declare(self, queue, exchange='', enable_ha=False, ttl=0): self._check_exception() if not self._connected: raise RuntimeError('Not connected to RabbitMQ') queue_arguments = {} if enable_ha is True: # To use mirrored queues feature in RabbitMQ 2.x # we need to declare this policy on the queue itself. # # Warning: this option has no effect on RabbitMQ 3.X, # to enable mirrored queues feature in RabbitMQ 3.X, please # configure RabbitMQ. queue_arguments['x-ha-policy'] = 'all' if ttl > 0: queue_arguments['x-expires'] = ttl exchange = kombu.Exchange(exchange, type='direct', durable=True) queue = kombu.Queue(queue, exchange, queue, durable=False, queue_arguments=queue_arguments) bound_queue = queue(self._connection) bound_queue.declare()
def _create_exchange_and_queue(queue_name): """ Create a Kombu message Exchange and Queue. Args: queue_name (str): The target queue's name Returns: tuple(kombu.Exchange, kombu.Queue) """ exchange = kombu.Exchange( settings.QUEUE_EXCHANGE_NAME, 'direct', durable=True ) message_queue = kombu.Queue( queue_name, exchange=exchange, routing_key=queue_name ) return exchange, message_queue
def publish(self, message: Union[dict, str], headers: dict = None, exchange: str = 'transport', routing_key: str = '*', exchange_type: Exchange_Type = 'direct') -> None: """ Publish a message to th AMQP Queue :param message: message to be published :param headers: header key-values to publish with the message :param exchange: specifies the top level specifier for message publish :param routing_key: determines which queue the message is published to :param exchange_type: Type of the exchange - ['direct', 'fanout', 'headers', 'topic'] """ self._conn.connect() producer = kombu.Producer(self._conn.channel()) producer.publish(message, headers=headers or {}, exchange=kombu.Exchange(exchange, type=exchange_type), routing_key=routing_key) producer.close() self._conn.release()
def _publish_consume(self): results = [] def process_message(body, message): results.append(body) message.ack() task_queue = kombu.Queue('tasks', kombu.Exchange('tasks'), routing_key='tasks') to_publish = {'hello': 'world'} self.producer.publish(to_publish, exchange=task_queue.exchange, routing_key=task_queue.routing_key, declare=[task_queue]) with kombu.Consumer(self.conn, [task_queue], accept=['json'], callbacks=[process_message]) as consumer: Pin.override(consumer, service='kombu-patch', tracer=self.tracer) self.conn.drain_events(timeout=2) self.assertEqual(results[0], to_publish)
def __init__(self, hosts, \ queue, routing_key, exchange_type='direct', \ serializer='json', compression='zlib'): self.hosts = hosts self.queue = queue self.routing_key = routing_key self.exchange_type = exchange_type self.serializer = serializer self.compression = compression self.connection = kombu.Connection(hosts) self.connection.ensure_connection() self.connection.connect() self.channel = self.connection.channel() self.exchange = kombu.Exchange(self.queue, type=exchange_type) #self.queue = kombu.Queue(self.queue, self.exchange, \ # routing_key=self.routing_key) self.producer = kombu.Producer(self.channel, \ self.exchange, routing_key=self.routing_key, \ serializer=self.serializer, \ compression=self.compression, \ on_return=on_return_callback)
def publish(self, message: Union[dict, str] = "", headers: dict = None, exchange: str = EXCHANGE, routing_key: str = ROUTING_KEY): """ Publish a message to th AMQP Queue :param message: message to be published :param headers: header key-values to publish with the message :param exchange: specifies the top level specifier for message publish :param routing_key: determines which queue the message is published to """ self._conn.connect() queue = kombu.Queue(routing_key, kombu.Exchange(exchange, type="topic"), routing_key=routing_key) queue.maybe_bind(self._conn) queue.declare() producer = kombu.Producer(self._conn.channel()) producer.publish( message, headers=headers or {}, exchange=queue.exchange, routing_key=queue.routing_key, declare=[queue] ) producer.close() self._conn.release()
def reload_data(instance_config, job_id): """ reload data on all kraken of this instance""" job = models.Job.query.get(job_id) instance = job.instance logging.info("Unqueuing job {}, reload data of instance {}".format(job.id, instance.name)) logger = get_instance_logger(instance) try: task = navitiacommon.task_pb2.Task() task.action = navitiacommon.task_pb2.RELOAD connection = kombu.Connection(current_app.config['CELERY_BROKER_URL']) exchange = kombu.Exchange(instance_config.exchange, 'topic', durable=True) producer = connection.Producer(exchange=exchange) logger.info("reload kraken") producer.publish(task.SerializeToString(), routing_key=instance.name + '.task.reload') connection.release() except: logger.exception('') job.state = 'failed' models.db.session.commit() raise
def __init__( self, url, manager_name, connect_ssl=None, timeout=DEFAULT_TIMEOUT, publish_kwds={}, ): """ """ if not kombu: raise Exception(KOMBU_UNAVAILABLE) self.__url = url self.__manager_name = manager_name self.__connect_ssl = connect_ssl self.__exchange = kombu.Exchange(DEFAULT_EXCHANGE_NAME, DEFAULT_EXCHANGE_TYPE) self.__timeout = timeout # Be sure to log message publishing failures. if publish_kwds.get("retry", False): if "retry_policy" not in publish_kwds: publish_kwds["retry_policy"] = {} if "errback" not in publish_kwds["retry_policy"]: publish_kwds["retry_policy"]["errback"] = self.__publish_errback self.__publish_kwds = publish_kwds
def __init__(self, url, queue, headers, allowed, username='******', password='******', **kwargs): super(KombuPublisher, self).__init__(allowed, **kwargs) self._url = url self.queue = queue self._headers = headers self.username = username self.password = password self.exchange = kombu.Exchange(name='amq.direct', type='direct') self._queue = kombu.Queue(name=queue, exchange=self.exchange, routing_key=queue) self.connection = kombu.Connection(self._url, userid=self.username, password=self.password) self.producer = kombu.Producer(self.connection, routing_key=self.queue, exchange=self.exchange)
def amqp_publish_user(owner, routing_key, data): with kombu.Connection(config.BROKER_URL) as connection: channel = connection.channel() try: kombu.Producer(channel).publish( data, exchange=kombu.Exchange(_amqp_owner_exchange(owner)), routing_key=routing_key, serializer='json', retry=True ) started_at = time() while True: try: connection.drain_events(timeout=0.5) except AmqpNotFound: raise except: pass if time() - started_at >= 0.5: break except AmqpNotFound: return False else: return True finally: channel.close()
def test_user_published(self): idm_broker_config = apps.get_app_config('forsta_broker') with idm_broker_config.broker.acquire(block=True) as conn: queue = kombu.Queue(exclusive=True).bind(conn) queue.declare() queue.bind_to(exchange=kombu.Exchange('idm.auth.user'), routing_key='#') connection = transaction.get_connection() self.assertFalse(connection.in_atomic_block) with transaction.atomic(): user = User.objects.create(identity_id=uuid.uuid4(), primary=True, is_active=True) for i in range(5): message = queue.get() if message: break time.sleep(0.1) self.assertIsInstance(message, Message) self.assertEqual(message.delivery_info['routing_key'], 'User.created.{}'.format(str(user.id))) self.assertEqual(message.content_type, 'application/json') self.assertEqual( json.loads(message.body.decode())['@type'], 'User')
def get_listener(conn, userid, exchanges=None, extra_data=None, logger=None): """Obtain a Pulse consumer that can handle received messages. Returns a ``Listener`` instance bound to listen to the requested exchanges. Callers should use ``add_callback`` to register functions that will be called when a message is received. The callback functions receive one argument ``body``, the decoded message body. """ queues = [] if exchanges is None: raise ValueError("No exchanges supplied") for queue_name, exchange_name, key_name in exchanges: queue_name = 'queue/%s/%s' % (userid, queue_name) exchange = kombu.Exchange(exchange_name, type='topic', channel=conn) exchange.declare(passive=True) queue = kombu.Queue(name=queue_name, exchange=exchange, durable=True, routing_key=key_name, exclusive=False, auto_delete=False, channel=conn, extra_data=extra_data) queues.append(queue) # queue.declare() declares the exchange, which isn't allowed by the # server. So call the low-level APIs to only declare the queue itself. queue.queue_declare() queue.queue_bind() return Listener(conn, [item[1] for item in exchanges], queues, logger)
def _make_exchange(name, durable=False, auto_delete=True, exchange_type='topic'): """Make named exchange. This method creates object representing exchange on RabbitMQ. It would create a new exchange if exchange with given name don't exists. :param name: Name of the exchange. :param durable: If set to True, messages on this exchange would be store on disk - therefore can be retrieve after failure. :param auto_delete: If set to True, exchange would be automatically deleted when none is connected. :param exchange_type: Type of the exchange. Can be one of 'direct', 'topic', 'fanout', 'headers'. See Kombu docs for further details. :return: Kombu exchange object. """ return kombu.Exchange(name=name, type=exchange_type, durable=durable, auto_delete=auto_delete)
def amqp_subscribe(exchange, callback, queue='', ex_type='fanout', routing_keys=None, durable=False, auto_delete=True): with kombu.pools.connections[kombu.Connection(config.BROKER_URL)].acquire( block=True, timeout=10) as connection: exchange = kombu.Exchange(exchange, type=ex_type, durable=durable, auto_delete=auto_delete) if not routing_keys: queue = kombu.Queue(queue, exchange, exclusive=True) else: queue = kombu.Queue(queue, [ kombu.binding(exchange, routing_key=key) for key in routing_keys ], exclusive=True) with connection.Consumer([queue], callbacks=[callback], no_ack=True): while True: connection.drain_events()
def run(connection, args): channel = connection.channel() exchange = kombu.Exchange('villas', type='headers', durable=True) producer = kombu.Producer(channel, exchange=exchange) message = {'action': 'start'} if args.parameters is not None: message['parameters'] = _get_parameters(args.parameters, args.parameters_file) try: if args.model is not None: message['model'] = _get_parameters(args.model, args.model_file) if args.results is not None: message['results'] = _get_parameters(args.results, args.results_file) except yaml.YAMLError as e: LOGGER.error('Failed to parse parameters: %s at line %d column %d', e.msg, e.lineno, e.colno) producer.publish(message, headers=SimulatorCommand.get_headers(args))
def __init__(self, url=None, prefix='asgi:', expiry=60, group_expiry=86400, capacity=100, channel_capacity=None): try: init_func = import_string(self.config["INIT_FUNC"]) init_func() except KeyError: pass except ImportError: raise RuntimeError("Cannot import INIT_FUNC") try: self.model = import_string(self.config["MODEL"]) except KeyError: from .models import ChannelGroup self.model = ChannelGroup except ImportError: raise RuntimeError("Cannot import MODEL") super(AMQPChannelLayer, self).__init__( expiry=expiry, group_expiry=group_expiry, capacity=capacity, channel_capacity=channel_capacity, ) self.url = url or 'amqp://*****:*****@localhost:5672/%2F' self.prefix = prefix + 'tower:websocket' self.exchange = kombu.Exchange(self.prefix, type='topic') self.tdata = threading.local()
def __init__(self, cwuser: str, cwpass: str, workers: int = 1): """ Инициализация класса для работы с АПИ :param cwuser: str - API username :param cwpass: str - API password :param workers: int - Number of workers to use """ # TODO Разобраться с пулом работников (нельзя использовать 1 канал на всех) self.__lock = threading.Lock() self.lock = threading.Condition(self.__lock) self.cwuser = cwuser self.cwpass = cwpass self.url = f'amqps://{cwuser}:{cwpass}@api.chtwrs.com:5673' self.connected = False # Соединение активно в данный момент self.connecting = False # True, если соединение не установлено, но пытается установиться в данный момент self.active = True # True при запуске, и False в самом конце, если self.active == True и # # self.connected == False, то это значит, что соединение оборвалось само. self.kafka_active = False self.guild_changes = {} self.guild_changes_work = None self.conn = None self.cursor = None self.connection = None self.producer = None self.bot = dispatcher.bot self.consumer_tags = [] self.num_workers = workers # Число работников, работающих над отправкой запросов self.workers = [] # Сами работники self.requests_queue = Queue() # Очередь с запросами (Dict) self.__requests_per_second = 0 # Счётчик запросов в секунду self.EXCHANGE = "{}_ex".format(cwuser) self.ROUTING_KEY = "{}_o".format(cwuser) self.INBOUND = "{}_i".format(self.cwuser) self.exchange = kombu.Exchange(self.EXCHANGE) self.inbound_queue = kombu.Queue(self.INBOUND) self.kafka_consumer = None self.sent = 0 self.got_responses = 0 self.callbacks = { "createAuthCode": self.on_create_auth_code, "grantToken": self.on_grant_token, "requestProfile": self.on_request_profile, "guildInfo": self.on_guild_info, "requestGearInfo": self.on_gear_info, "authAdditionalOperation": self.on_request_additional_operation, "grantAdditionalOperation": self.on_grant_additional_operational, "requestStock": self.on_stock_info, 'cw3-deals': self.on_deals, # 'cw3-offers': self.on_offers, # not implemented 'cw3-sex_digest': self.on_sex_digest, 'cw3-yellow_pages': self.on_yellow_pages, # 'cw3-au_digest': self.on_au_digest, # not implemented }