class MessageHandler: def __init__(self): self.conn = Connection('amqp://*****:*****@localhost:5672//') def add_message(self, message, queue_name): simple_queue = self.conn.SimpleQueue(queue_name) simple_queue.put(message) simple_queue.close() def get_message(self, queue_name): simple_queue = self.conn.SimpleQueue(queue_name) message = simple_queue.get(block=True, timeout=1) message.ack() simple_queue.close() return message
def __init__(self, uri=None, host_url='localhost', queue='logging'): logging.Handler.__init__(self) if uri: connection = Connection(uri) self.queue = connection.SimpleQueue(queue) self.host = host_url
def inner(*args, **kwargs): connection = Connection(settings.BROKER_URL) connection.connect() queue = connection.SimpleQueue(settings.QUEUE_NAME) res = func(*args, **kwargs, queue=queue) connection.release() return res
class CeleryQueue(QueueBase): """ At some point this will adapt `django-signalqueue` for use with popular `(dj)celery` platform (but not today). When this class is done, I will discuss it here. """ def __init__(self, *args, **kwargs): super(CeleryQueue, self).__init__(*args, **kwargs) self.celery_queue_name = self.queue_options.pop( 'celery_queue_name', 'inactive') self.serializer = self.queue_options.pop('serializer', 'json') self.compression = self.queue_options.pop('compression', None) self.kc = Connection(**self.queue_options) self.kc.connect() self.qc = self.kc.SimpleQueue(name=self.celery_queue_name) def ping(self): return self.kc.connected and not self.qc.channel.closed def push(self, value): self.qc.put(value, compression=self.compression, serializer=None) def pop(self): virtual_message = self.qc.get(block=False, timeout=1) return virtual_message.payload def count(self): try: return self.qc.qsize() except kombu.exceptions.StdChannelError: self.qc.queue.declare() return 0 def clear(self): self.qc.clear() def values(self, **kwargs): return [] def __getitem__(self, idx): #return self.values().__getitem__(idx) return '' def dispatch(self, signal, sender=None, **kwargs): task_name = "%s:%s" % (signal.regkey, signal.name) try: result = tasks[task_name].delay(sender=sender, **kwargs) except tasks.NotRegistered: pass else: return result
class KombuMailbox(AckableMailbox): def __init__(self, address, name, transport_options, ssl=False, no_ack=True, queue_opts=None, exchange_opts=None): from kombu import Connection self._conn = Connection(address, transport_options=transport_options, ssl=ssl) self._queue = self._conn.SimpleQueue(name, no_ack, queue_opts, exchange_opts) self._no_ack = no_ack self._last_msg = None def get(self): self._last_msg = self._queue.get() return decode( unpackb(self._last_msg.body, encoding='utf-8', use_list=False)) def put(self, message): return self._queue.put( packb(encode(message), encoding='utf-8', use_bin_type=True)) def ack(self): if self._no_ack: return if self._last_msg is not None: self._last_msg.ack() self._last_msg = None def encode(self): raise NotImplementedError @staticmethod def decode(params): raise NotImplementedError def __enter__(self): return self def __exit__(self, *exc_details): self.__del__() def __del__(self): if hasattr(self, '_queue'): self._queue.close() if hasattr(self, '_conn'): self._conn.close()
def send_task(obj, message): log.info('send message: %s' % message['event_type']) try: connection = Connection(PLAYOUT_BROKER_URL) simple_queue = connection.SimpleQueue(BROKER_QUEUE) simple_queue.put(json.dumps(message)) simple_queue.close() connection.close() except Exception as e: log.error('error sending message: %s' % e)
def reset(self): logging.debug("Attempting kombu reset: %s" % self.uri) conn = Connection(self.uri, connect_timeout=.5) try: logging.debug("Connected ... creating queue: %s" % self.queue_name) q = conn.SimpleQueue(self.queue_name) while True: logging.debug("Attempting to read msg from queue ...") m = q.get(block=False) m.ack() except Queue.Empty: pass logging.debug("Done with purge ...")
def connect_queue(QUEUE_USERNAME, QUEUE_PASSWORD, QUEUE_HOST, QUEUE_VIRTUAL_HOST, QUEUE): #example of parameter #QUEUE_USERNAME='******' #QUEUE_PASSWORD='******' #QUEUE_HOST='queue.platform.acommercedev.com' #QUEUE_VIRTUAL_HOST='myvhost' #QUEUE='partner_master' url = "amqp://{0}:{1}@{2}:5672/{3}".format(QUEUE_USERNAME, QUEUE_PASSWORD, QUEUE_HOST, QUEUE_VIRTUAL_HOST) print("url :") + url #### open connection to QUEUE#### con = Connection(url) msg_queue = con.SimpleQueue(QUEUE) return msg_queue
def __init__(self, crawl_name, num_urls=DEFAULT_NUM_URLS): """ Create a NutchUrlTrails instance for visualizing a running Nutch crawl in real-time using Bokeh :param name: The name of the crawl (as identified by the queue) :param num_urls: The number of URLs to display in the visualization :return: A NutchUrLTrails instance """ self.crawl_name = crawl_name self.num_urls = num_urls self.open_urls = {} self.closed_urls = {} self.old_segments = None self.old_circles = None self.session = Session() self.session.use_doc(self.crawl_name) self.document = Document() con = Connection() exchange = Exchange(EXCHANGE_NAME, 'direct', durable=False) queue = Queue(crawl_name, exchange=exchange, routing_key=crawl_name) self.queue = con.SimpleQueue(name=queue)
class CommissaireService(ConsumerMixin, BusMixin): """ Commissaire service class. """ #: Default configuration file if none is specified. Subclasses #: should override this. _default_config_file = C.DEFAULT_CONFIGURATION_FILE def __init__(self, exchange_name, connection_url, qkwargs, config_file=None): """ Initializes a new Service instance. :param exchange_name: Name of the topic exchange. :type exchange_name: str :param connection_url: Kombu connection url. :type connection_url: str :param qkwargs: One or more dicts keyword arguments for queue creation :type qkwargs: list :param config_file: Path to the configuration file location. :type config_file: str or None """ name = self.__class__.__name__ self.logger = logging.getLogger(name) self.logger.debug('Initializing {}'.format(name)) # If we are given no default, use the global one # Read the configuration file self._config_data = read_config_file(config_file, self._default_config_file) if connection_url is None and 'bus_uri' in self._config_data: connection_url = self._config_data.get('bus_uri') self.logger.debug('Using connection_url=%s from config file', connection_url) if exchange_name is None and 'exchange_name' in self._config_data: self.logger.debug('Using exchange_name=%s from config file', exchange_name) exchange_name = self._config_data.get('bus_exchange') self.connection = Connection(connection_url) self._channel = self.connection.default_channel self._exchange = Exchange(exchange_name, type='topic').bind(self._channel) self._exchange.declare() # Set up queues self._queues = [] for kwargs in qkwargs: queue = Queue(**kwargs) queue.exchange = self._exchange queue = queue.bind(self._channel) self._queues.append(queue) self.logger.debug(queue.as_dict()) # Create producer for publishing on topics self.producer = Producer(self._channel, self._exchange) self.logger.debug('Initializing of {} finished'.format(name)) def get_consumers(self, Consumer, channel): """ Returns the a list of consumers to watch. Called by the parent Mixin. :param Consumer: Message consumer class. :type Consumer: kombu.Consumer :param channel: An opened channel. :type channel: kombu.transport.*.Channel :returns: A list of Consumer instances. :rtype: list """ consumers = [] self.logger.debug('Setting up consumers') for queue in self._queues: self.logger.debug('Will consume on {}'.format(queue.name)) consumers.append(Consumer(queue, callbacks=[self.on_message])) self.logger.debug('Consumers: {}'.format(consumers)) return consumers def on_message(self, body, message): """ Called when a new message arrives. :param body: Body of the message. :type body: dict or json string :param message: The message instance. :type message: kombu.message.Message """ self.logger.debug('Received message "{}" {}'.format( message.delivery_tag, body)) expected_method = message.delivery_info['routing_key'].rsplit('.', 1)[1] # If we don't get a valid message we default to -1 for the id response = {'jsonrpc': '2.0', 'id': -1} try: # If we don't have a dict then it should be a json string if isinstance(body, str): body = json.loads(body) # If we have a method and it matches the routing key treat it # as a jsonrpc call if (isinstance(body, dict) and 'method' in body.keys() and body.get('method') == expected_method): response['id'] = body.get('id', -1) method = getattr(self, 'on_{}'.format(body['method'])) if type(body.setdefault('params', {})) is dict: result = method(message=message, **body['params']) else: result = method(message, *body['params']) response['result'] = result self.logger.debug('Result for "{}": "{}"'.format( response['id'], result)) else: # Drop it self.logger.error('Dropping unknown message: payload="{}", ' 'properties="{}"'.format( body, message.properties)) except Exception as error: # Subclasses of RemoteProcedureCallError are re-created and # raised on the client-side. if isinstance(error, RemoteProcedureCallError): response['error'] = { 'code': error.code, 'message': str(error), 'data': error.data } else: jsonrpc_error_code = C.JSONRPC_ERRORS['INVALID_REQUEST'] # If there is an attribute error then use the Method Not Found # code in the error response if type(error) is AttributeError: jsonrpc_error_code = C.JSONRPC_ERRORS['METHOD_NOT_FOUND'] elif type(error) is json.decoder.JSONDecodeError: jsonrpc_error_code = C.JSONRPC_ERRORS['INVALID_JSON'] response['error'] = { 'code': jsonrpc_error_code, 'message': str(error), 'data': { 'exception': str(type(error)) } } self.logger.warn( 'Exception raised during method call:\n{}'.format( traceback.format_exc())) # Reply back if needed if message.properties.get('reply_to'): self.logger.debug('Responding to {}'.format( message.properties['reply_to'])) response_queue = self.connection.SimpleQueue( message.properties['reply_to']) response_queue.put(json.dumps(response)) response_queue.close() message.ack() self.logger.debug('Message "{}" {} ackd'.format( message.delivery_tag, ('was' if message.acknowledged else 'was not'))) def respond(self, queue_name, id, payload, **kwargs): """ Sends a response to a simple queue. Responses are sent back to a request and never should be the owner of the queue. :param queue_name: The name of the queue to use. :type queue_name: str :param id: The unique request id :type id: str :param payload: The content of the message. :type payload: dict :param kwargs: Keyword arguments to pass to SimpleQueue :type kwargs: dict """ self.logger.debug('Sending response for message id "{}"'.format(id)) send_queue = self.connection.SimpleQueue(queue_name, **kwargs) jsonrpc_msg = { 'jsonrpc': "2.0", 'id': id, 'result': payload, } self.logger.debug('jsonrpc msg: {}'.format(jsonrpc_msg)) send_queue.put(jsonrpc_msg) self.logger.debug('Sent response for message id "{}"'.format(id)) send_queue.close() def onconnection_revived(self): # pragma: no cover """ Called when a reconnection occurs. """ self.logger.info('Connection (re)established') def on_consume_ready(self, connection, channel, consumers): # pragma: no cover """ Called when the service is ready to consume messages. :param connection: The current connection instance. :type connection: kombu.Connection :param channel: The current channel. :type channel: kombu.transport.*.Channel :param consumers: A list of consumers. :type consumers: list """ self.logger.info('Ready to consume') if self.logger.level == logging.DEBUG: queue_names = [] for consumer in consumers: queue_names += [x.name for x in consumer.queues] self.logger.debug( 'Consuming via connection "{}" and channel "{}" on ' 'the following queues: "{}"'.format(connection.as_uri(), channel, '", "'.join(queue_names))) def on_consume_end(self, connection, channel): # pragma: no cover """ Called when the service stops consuming. :param connection: The current connection instance. :type connection: kombu.Connection :param channel: The current channel. :type channel: kombu.transport.*.Channel """ self.logger.warn('Consuming has ended')
class Queue(object): def __init__(self, url=None, hostname=None, userid=None, password=None, queue_name="basic_queue", serializer="pickle", compression=None): # current_queue self.is_local = True self.localized = None self.distributed = None if (hostname is None or userid is None or password is None) and url is None: self.localized = multiprocessing.Queue() else: self.is_local = False self.serializer = serializer self.compression = compression self.connection = Connection(url, heartbeat=5) self.distributed = self.connection.SimpleQueue(queue_name) def __len__(self): return self.qsize() def join(self): pass def qsize(self) -> int: if self.is_local == True: return self.localized.qsize() return self.distributed.qsize() def clear_queue(self): if self.is_local: while not self.empty(): self.get_nowait() else: self.distributed.clear() def empty(self) -> bool: if self.is_local: return self.localized.empty() else: current_size = self.distributed.qsize() if current_size == 0: return True return False def put(self, item, block=True, timeout=None): if self.is_local == True: self.localized.put(item, block=block, timeout=timeout) else: self.distributed.put(item, serializer=self.serializer, compression=self.compression) def get(self, block=True, timeout=None): if self.is_local == True: return self.localized.get(block=block, timeout=timeout) else: return self.distributed.get() def put_nowait(self, item): if self.is_local == True: self.localized.put_nowait(item) else: self.distributed.put_nowait(item, serializer=self.serializer, compression=self.compression) def get_nowait(self): if self.is_local == True: return self.localized.get_nowait() else: return self.distributed.get_nowait() def close(self): """ Closes the distributed queue if it's distributed with redis or any other library. """ if self.is_local == False: self.distributed.close()
class Bus(BusMixin): """ Connection to a bus. """ def __init__(self, exchange_name, connection_url, qkwargs): """ Initializes a new Bus instance. :param exchange_name: Name of the topic exchange. :type exchange_name: str :param connection_url: Kombu connection url. :type connection_url: str :param qkwargs: One or more dicts keyword arguments for queue creation :type qkwargs: list """ self.logger = logging.getLogger('Bus') self.logger.debug('Initializing bus connection') self.connection = None self._channel = None self._exchange = None self.exchange_name = exchange_name self.connection_url = connection_url self.qkwargs = qkwargs self.storage = StorageClient(self) @property def init_kwargs(self): """ Returns the initializing kwargs for this instance. :returns: the initializing kwargs for this instance. :rtype: dict """ return { 'exchange_name': self.exchange_name, 'connection_url': self.connection_url, 'qkwargs': self.qkwargs, } def connect(self): """ 'Connects' to the bus. :returns: The same instance. :rtype: commissaire_http.bus.Bus """ if self.connection is not None: self.logger.warn('Bus already connected.') return self self.connection = Connection(self.connection_url) self._channel = self.connection.channel() self._exchange = Exchange( self.exchange_name, type='topic').bind(self._channel) self._exchange.declare() # Create queues self._queues = [] for kwargs in self.qkwargs: queue = Queue(**kwargs) queue.exchange = self._exchange queue = queue.bind(self._channel) self._queues.append(queue) self.logger.debug('Created queue {}'.format(queue.as_dict())) # Create producer for publishing on topics self.producer = Producer(self._channel, self._exchange) self.logger.debug('Bus connection finished') return self def respond(self, queue_name, id, payload, **kwargs): # pragma: no cover """ Sends a response to a simple queue. Responses are sent back to a request and never should be the owner of the queue. :param queue_name: The name of the queue to use. :type queue_name: str :param id: The unique request id :type id: str :param payload: The content of the message. :type payload: dict :param kwargs: Keyword arguments to pass to SimpleQueue :type kwargs: dict """ self.logger.debug('Sending response for message id "{}"'.format(id)) send_queue = self.connection.SimpleQueue(queue_name, **kwargs) jsonrpc_msg = { 'jsonrpc': "2.0", 'id': id, 'result': payload, } self.logger.debug('jsonrpc msg: {}'.format(jsonrpc_msg)) send_queue.put(jsonrpc_msg) self.logger.debug('Sent response for message id "{}"'.format(id)) send_queue.close()
def setup(): global conn, q conn = Connection(connectionString) conn.connect() q = conn.SimpleQueue(queueName)
if settings.MOESIF_MIDDLEWARE.get('USE_CELERY', False): if BROKER_URL: try: from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger import atexit from kombu import Connection from kombu.exceptions import ChannelError scheduler = BackgroundScheduler(daemon=True) scheduler.start() config, config_etag, sampling_percentage, last_updated_time = get_config( ) try: conn = Connection(BROKER_URL) moesif_events_queue = conn.SimpleQueue('moesif_events_queue') scheduler.add_job( func=lambda: async_client_create_event( moesif_events_queue, config, config_etag, last_updated_time), trigger=IntervalTrigger(seconds=5), id='moesif_events_batch_job', name='Schedule events batch job every 5 second', replace_existing=True) # Exit handler when exiting the app atexit.register( lambda: exit_handler(moesif_events_queue, scheduler)) except: if DEBUG: print("Error while connecting to - {0}".format(BROKER_URL))
class KombuQueue(object): """ kombu is a high-level interface for multiple message queue backends. KombuQueue is built on top of kombu API. """ Empty = BaseQueue.Empty Full = BaseQueue.Full max_timeout = 0.3 def __init__(self, name, url="amqp://", maxsize=0, lazy_limit=True): """ Constructor for KombuQueue url: http://kombu.readthedocs.org/en/latest/userguide/connections.html#urls maxsize: an integer that sets the upperbound limit on the number of items that can be placed in the queue. """ self.name = name self.conn = Connection(url) self.queue = self.conn.SimpleQueue(self.name, no_ack=True, serializer='umsgpack') self.maxsize = maxsize self.lazy_limit = lazy_limit if self.lazy_limit and self.maxsize: self.qsize_diff_limit = int(self.maxsize * 0.1) else: self.qsize_diff_limit = 0 self.qsize_diff = 0 def qsize(self): try: return self.queue.qsize() except ChannelError: return 0 def empty(self): if self.qsize() == 0: return True else: return False def full(self): if self.maxsize and self.qsize() >= self.maxsize: return True else: return False def put(self, obj, block=True, timeout=None): if not block: return self.put_nowait() start_time = time.time() while True: try: return self.put_nowait(obj) except BaseQueue.Full: if timeout: lasted = time.time() - start_time if timeout > lasted: time.sleep(min(self.max_timeout, timeout - lasted)) else: raise else: time.sleep(self.max_timeout) def put_nowait(self, obj): if self.lazy_limit and self.qsize_diff < self.qsize_diff_limit: pass elif self.full(): raise BaseQueue.Full else: self.qsize_diff = 0 return self.queue.put(obj) def get(self, block=True, timeout=None): try: ret = self.queue.get(block, timeout) return ret.payload except self.queue.Empty: raise BaseQueue.Empty def get_nowait(self): try: ret = self.queue.get_nowait() return ret.payload except self.queue.Empty: raise BaseQueue.Empty def delete(self): self.queue.queue.delete() def __del__(self): self.queue.close()
from kombu import Connection conn = Connection("amqp://*****:*****@localhost:5672/") queue = conn.SimpleQueue('example1') queue.put("test from producer") conn.close()
def __init__(self, uri=None, queue='logging'): logging.Handler.__init__(self) if uri: connection = Connection(uri) self.queue = connection.SimpleQueue(queue)
sys.path.append(os.getcwd() + "/..") from scraper import scraper from repo import db_repo CYCLONE_LIST_QUEUE = 'cyclones.list' CYCLONE_DETAILS_QUEUE = 'cyclones.details' CYCLONE_HISTORY_QUEUE = "cyclone.history" CYCLONE_FORECAST_QUEUE = "cyclone.forecast" BROKER_URL = "redis://redis" kombu_conn = Connection(BROKER_URL) list_queue = kombu_conn.SimpleQueue(CYCLONE_LIST_QUEUE) details_queue = kombu_conn.SimpleQueue(CYCLONE_DETAILS_QUEUE) history_queue = kombu_conn.SimpleQueue(CYCLONE_HISTORY_QUEUE) forecast_queue = kombu_conn.SimpleQueue(CYCLONE_FORECAST_QUEUE) CELERY_BROKER_URL = 'redis://redis' app = Celery('tasks', broker=CELERY_BROKER_URL) @app.task def extract_cyclones_list(): log.info("extract list of cyclones") cyclones = scraper.extract_cyclones_list() for c in cyclones: log.info("adding %s", c)
class StreamArchiver(LogMixin, NotificationMixin, StreamListener): def __init__(self, archives, verbosity=1, *args, **kwargs): super().__init__(*args, **kwargs) # A temporary storage for use in exception forensics self.raw_data = None # All archives in a stream belong to the same user self.user = archives[0].user self.verbosity = verbosity self.connection = Connection(settings.BROKER_URL) self.channels = [] for archive in archives: with Connection(settings.BROKER_URL) as connection: consumer = ArchiveConsumer(archive, connection, verbosity=self.verbosity) consumer.start() self.channels.append({ "archive": archive, "consumer": consumer, "queue": self.connection.SimpleQueue("archiver:{}".format( archive.pk)), }) def set_verbosity(self, verbosity): self.logger.info("Setting {} stream verbosity to {}".format( self.user, verbosity)) self.verbosity = verbosity for channel in self.channels: channel["consumer"].set_verbosity(self.verbosity) def on_data(self, raw_data): self.raw_data = raw_data return StreamListener.on_data(self, raw_data) def on_status(self, status): for channel in self.channels: archive = channel["archive"] queue = channel["queue"] query = archive.query.lower() if query in status.text.lower(): queue.put(status._json) elif hasattr(status, "retweeted_status"): if query in status.retweeted_status.text.lower(): queue.put(status._json) elif hasattr(status.retweeted_status, "quoted_status"): if query in status.retweeted_status.quoted_status[ "text"].lower(): # NOQA: E501 queue.put(status._json) elif hasattr(status, "quoted_status"): if query in status.quoted_status["text"].lower(): queue.put(status._json) def on_exception(self, exception): additional = "Source: {}".format(self.raw_data) self._alert("Collector exception [listener]", exception, additional) stderr.write("\n\nEXCEPTION:\n{}\n\nSource: {}\n".format( exception, additional)) self.close_log() return False def on_error(self, status_code): message = str(status_code) if status_code == 401: message = ( f"Twitter issued a 401 for {self.user}, so they've been " f"kicked.") self.user.status = User.STATUS_DISABLED self.user.save(update_fields=("status", )) self._alert("Collector Twitter error", message) stderr.write("ERROR: Twitter responded with {}".format(status_code)) self.close_log() return False def on_disconnect(self, notice): """ This is what happens if *Twitter* sends a disconnect, not if we disconnect from the stream ourselves. """ self._alert("Collector disconnect", str(notice)) stderr.write("\n\nTwitter disconnect: {}\n\n\n".format(notice)) self.close_log() return False def close_log(self): self.connection.close() self.connection.release() # Set `should_stop` which queues the consumer to close everything up for channel in self.channels: channel["consumer"].should_stop = True # Now wait until the consumer has confirmed that it's finished for channel in self.channels: while not channel["consumer"].is_stopped: if self.verbosity > 1: self.logger.info("Waiting for {} to stop".format( channel["consumer"].archive)) sleep(0.1) stdout.flush() Archive.objects.filter( pk__in=[__["archive"].pk for __ in self.channels]).update(is_running=False)
class KomBuClient(ConsumerMixin): def __init__(self, hosts_conf, exchange_name='', exchange_type='', exchange_arguments=None, queue_name='', routing_key='', queue_arguments=None, callback=None, no_ack=True): self.hosts_conf = hosts_conf self.hosts = self.create_hosts() self.connection = Connection(self.hosts) self.task_exchange = Exchange(name=exchange_name, type=exchange_type, arguments=exchange_arguments) self.task_queues = [Queue(name=queue_name, exchange=self.task_exchange, routing_key=routing_key, queue_arguments=queue_arguments)] self.callback = callback self.no_ack = no_ack def queue_size(self, queue_list, queue_arguments=None): result = dict() for i in queue_list: queue_size = self.connection.SimpleQueue(name=Queue(name=i, queue_arguments=queue_arguments)).qsize() result[i] = queue_size return result def create_hosts(self): hosts_list = [] for i in self.hosts_conf: host = i.get('host', '127.0.0.1') port = i.get('port', '5672') username = i.get('username', 'guest') passwd = i.get('passwd', 'guest') auth = "amqp://{username}:{passwd}@{host}:{port}//".format(username=username, passwd=passwd, host=host, port=port) hosts_list.append(auth) return hosts_list def get_consumers(self, Consumer, channel): channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=False) return [Consumer(queues=self.task_queues, accept=['json', 'pickle', 'msgpack', 'yaml'], callbacks=[self.callback], no_ack=self.no_ack)] def process_task(self, body, message): print self.hosts print body, message.properties message.ack() def start(self): self.run() def send_task(self, payload, routing_key=None, priority=0, content_type=None, content_encoding=None, serializer=None, headers=None, compression=None, exchange=None, retry=False, retry_policy=None, declare=[], expiration=None): try: with producers[self.connection].acquire(block=True) as producer: producer.publish(payload, serializer=serializer, compression=compression, exchange=exchange, declare=declare, routing_key=routing_key, priority=priority, content_type=content_type, content_encoding=content_encoding, headers=headers, retry=retry, retry_policy=retry_policy, expiration=expiration) except Exception as error: return False, error return True, None def close(self): self.connection.close()
class connection: """ Class to connect to a BITCloud server. Kwargs: Kwrvals: See Also: Notes: Example: References: .. [1] """ TRANSPORT = 'amqp' VHOST = '/' RABBIT_PORT = 5672 MONGO_HOST = '193.136.222.234' MONGO_PORT = 27017 def __init__(self, host='localhost', user=None, passwd=None, queue='BiometricsQ', dbName='BiometricsExperiments'): """ Establish the connection to the server. Kwargs: Kwrvals: See Also: Notes: Example: References: .. [1] """ # connect to RabbitMQ server self.conn = Connection(hostname=host, port=self.RABBIT_PORT, transport=self.TRANSPORT, userid=user, password=passwd, virtual_host=self.VHOST) # declare queue (persistent queue) self.queue = self.conn.SimpleQueue(queue) # connect to MongoDB self.mongo_conn = pmg.Connection(self.MONGO_HOST, self.MONGO_PORT) self.mongo_db = self.mongo_conn[dbName] self.mongo_collection = self.mongo_db['experiments'] self.mongo_id = self.mongo_db['IDTracker'] # check if ID tracker exists res = self.mongo_id.find_one({'_id': 0}, {'_id': 1}) if res is None: self.mongo_id.insert({'_id': 0, 'nextID': 0}) # check if user exists in MongoDB collection = self.mongo_db['users'] doc = collection.find_one({'username': user}, {'_id': 1}) if doc is None: s = raw_input( "Please register your E-Mail to be notified of task completion:\n" ) while True: if s is '': collection.insert({'username': user}) print "User %s registered without E-Mail.\nYou will not be able to receive notifications." % user break elif '@' not in s: s = raw_input( "Badly formated E-Mail address - missing '@' character.\nYou provided %s.\nPlease retype the E-Mail address:\n" % s) elif '.' not in s.split('@')[1]: s = raw_input( "Badly formated E-Mail address - missing domain name.\nYou provided %s.\nPlease retype the E-Mail address:\n" % s) else: collection.insert({'username': user, 'email': s}) print "User %s registered with E-Mail %s." % (user, s) break # extra self things self.user = user def addTask(self, task=None): # check inputs if task is None: raise TypeError, "Please specify a task." # get a new experiment ID from MongoDB (atomic operation) expID = self.mongo_id.find_and_modify(query={'_id': 0}, update={'$inc': { 'nextID': 1 }})['nextID'] task.update({ '_id': expID, 'user': self.user, 'status': 'waiting', 'seen': False }) # save to MongoDB self.mongo_collection.insert(task) # confirm insert _ = self.mongo_collection.find_one({'_id': expID}, {'status': 1}) # publish to RabbitMQ self.queue.put({'_id': expID}) return expID def cancelTask(self, expID): # cancel a task (will be ignored by worker when delivered) ### update status checks on worker # update and get task qry = {'_id': expID} upd = { '$set': { 'user': self.user, 'status': 'finished', 'seen': True, 'results': None, 'worker': None, }, } self.mongo_collection.update(qry, upd) def requeue(self, expID): # requeue a task # update and get task qry = {'_id': expID} upd = {'$set': {'user': self.user, 'status': 'waiting', 'seen': False}} self.mongo_collection.update(qry, upd) # publish to RabbitMQ self.queue.put(qry) def listTasks(self, status='all', seen=None): # list tasks belonging to user # status = 'waiting' | 'running' | 'finished' | 'error' | 'all' search = {'user': self.user} # update the search if status is not 'all': search.update({'status': status}) if seen is not None: search.update({'seen': seen}) # perform the search doc = self.mongo_collection.find(search, {'_id': 1}) res = [item['_id'] for item in doc] return res def getTaskInfo(self, expID): # retrieve a task doc = self.mongo_collection.find_one({'_id': expID}) return doc def search(self, spec=None, fields=None): # search task DB return self.mongo_collection.find(spec, fields) def collectResults(self, expIDList, mapper, basePath, headerOrder=None): # collect the results of a list of experiments, selecting the items in mapper if headerOrder is None: # use default order headerOrder = ['starting_data', 'train', 'test', 'dimreduction'] # determine number of columns per row keyOrder = [] for item in headerOrder: try: aux = len(mapper[item]['items']) except KeyError: pass else: if aux > 0: keyOrder.append(item) rows = [] for item in expIDList: task = self.getTaskInfo(item) mrow = [] # task ID mrow.append(task['_id']) # general parameters for c in keyOrder: for pair in mapper[c]['items']: mrow.append(misc.slasherDict(task[c], pair[1])) # classifiers for name in mapper['classifier']['instances'].iterkeys(): for clf in mapper['classifier']['instances'][name]: row = copy.deepcopy(mrow) # append name row.append(name) # append parameters try: for pair in mapper['classifier']['items']: row.append( misc.slasherDict(task['classifier'][clf], pair[1])) except KeyError: # clf not present continue # load global results res = datamanager.skLoad( os.path.join(basePath, 'Exp-%d' % item, 'results', clf, 'results-global.dict')) EER = 100. * res['global']['authentication']['rates'][ 'EER'][0, 1] EID = 100. * res['global']['identification']['rates']['Err'] row.append(EER) row.append(EID) # append row rows.append(row) # build header header = [[], []] # task ID header[0].append('Task ID') header[1].append('') # general parameters for c in keyOrder: header[0].append(mapper[c]['name']) nb = len(mapper[c]['items']) header[0].extend((nb - 1) * ['']) for i in xrange(nb): header[1].append(mapper[c]['items'][i][0]) # classifier header[0].append(mapper['classifier']['name']) nb = len(mapper['classifier']['items']) header[0].extend((nb + 2) * ['']) # do not forget name, EER, EID header[1].append('Name') for i in xrange(nb): header[1].append(mapper['classifier']['items'][i][0]) header[1].append('EER (%)') header[1].append('EID (%)') return {'rows': rows, 'header': header} def markSeen(self, expID): # mark a task (or list of tasks) as seen if type(expID) is int: expID = [expID] elif type(expID) is list: pass else: raise TypeError, "The input must be of type int or list" for item in expID: self.mongo_collection.update({'_id': item}, {'$set': { 'seen': True }}) def close(self): # close connection to RabbitMQ self.conn.release() # close connection to MongoDB self.mongo_conn.close()
class CommissaireService(ConsumerMixin, BusMixin): """ An example prototype CommissaireService base class. """ def __init__(self, exchange_name, connection_url, qkwargs): """ Initializes a new Service instance. :param exchange_name: Name of the topic exchange. :type exchange_name: str :param connection_url: Kombu connection url. :type connection_url: str :param qkwargs: One or more dicts keyword arguments for queue creation :type qkwargs: list """ self.logger = logging.getLogger(self.__class__.__name__) self.logger.debug('Initializing {0}'.format(self.__class__.__name__)) self.connection = Connection(connection_url) self._channel = self.connection.channel() self._exchange = Exchange( exchange_name, type='topic').bind(self._channel) self._exchange.declare() # Set up queues self._queues = [] for kwargs in qkwargs: queue = Queue(**kwargs) queue.exchange = self._exchange queue = queue.bind(self._channel) self._queues.append(queue) self.logger.debug(queue.as_dict()) # Create producer for publishing on topics self.producer = Producer(self._channel, self._exchange) self.logger.debug('Initializing finished') def get_consumers(self, Consumer, channel): """ Returns the a list of consumers to watch. Called by the parent Mixin. :param Consumer: Message consumer class. :type Consumer: kombu.Consumer :param channel: An opened channel. :type channel: kombu.transport.*.Channel :returns: A list of Consumer instances. :rtype: list """ consumers = [] self.logger.debug('Setting up consumers') for queue in self._queues: self.logger.debug('Will consume on {0}'.format(queue.name)) consumers.append( Consumer(queue, callbacks=[self._wrap_on_message])) self.logger.debug('Consumers: {}'.format(consumers)) return consumers def on_message(self, body, message): """ Called when a non-jsonrpc message arrives. :param body: Body of the message. :type body: dict :param message: The message instance. :type message: kombu.message.Message """ self.logger.error( 'Dropping unknown message: payload="{}", properties="{}"'.format( body, message.properties)) def _wrap_on_message(self, body, message): """ Wraps on_message for jsonrpc routing and logging. :param body: Body of the message. :type body: dict or json string :param message: The message instance. :type message: kombu.message.Message """ self.logger.debug('Received message "{}" {}'.format( message.delivery_tag, body)) expected_method = message.delivery_info['routing_key'].rsplit( '.', 1)[1] # If we don't get a valid message we default to -1 for the id response = {'jsonrpc': '2.0', 'id': -1} try: # If we don't have a dict then it should be a json string if isinstance(body, str): body = json.loads(body) # If we have a method and it matches the routing key treat it # as a jsonrpc call if ( isinstance(body, dict) and 'method' in body.keys() and body.get('method') == expected_method): response['id'] = body.get('id', -1) method = getattr(self, 'on_{}'.format(body['method'])) if type(body.setdefault('params', {})) is dict: result = method(message=message, **body['params']) else: result = method(message, *body['params']) response['result'] = result self.logger.debug('Result for "{}": "{}"'.format( response['id'], result)) # Otherwise send it to on_message else: self.on_message(body, message) except Exception as error: jsonrpc_error_code = -32600 # If there is an attribute error then use the Method Not Found # code in the error response if type(error) is AttributeError: jsonrpc_error_code = -32601 elif type(error) is json.decoder.JSONDecodeError: jsonrpc_error_code = -32700 # Parser error response['error'] = { 'code': jsonrpc_error_code, 'message': str(error), 'data': { 'exception': str(type(error)) } } self.logger.warn( 'Exception raised during method call:\n{}'.format( traceback.format_exc())) # Reply back if needed if message.properties.get('reply_to'): self.logger.debug('Responding to {0}'.format( message.properties['reply_to'])) response_queue = self.connection.SimpleQueue( message.properties['reply_to']) response_queue.put(json.dumps(response)) response_queue.close() message.ack() self.logger.debug('Message "{0}" {1} ackd'.format( message.delivery_tag, ('was' if message.acknowledged else 'was not'))) def respond(self, queue_name, id, payload, **kwargs): """ Sends a response to a simple queue. Responses are sent back to a request and never should be the owner of the queue. :param queue_name: The name of the queue to use. :type queue_name: str :param id: The unique request id :type id: str :param payload: The content of the message. :type payload: dict :param kwargs: Keyword arguments to pass to SimpleQueue :type kwargs: dict """ self.logger.debug('Sending response for message id "{}"'.format(id)) send_queue = self.connection.SimpleQueue(queue_name, **kwargs) jsonrpc_msg = { 'jsonrpc': "2.0", 'id': id, 'result': payload, } self.logger.debug('jsonrpc msg: {}'.format(jsonrpc_msg)) send_queue.put(jsonrpc_msg) self.logger.debug('Sent response for message id "{}"'.format(id)) send_queue.close() def onconnection_revived(self): # pragma: no cover """ Called when a reconnection occurs. """ self.logger.info('Connection (re)established') def on_consume_ready( self, connection, channel, consumers): # pragma: no cover """ Called when the service is ready to consume messages. :param connection: The current connection instance. :type connection: kombu.Connection :param channel: The current channel. :type channel: kombu.transport.*.Channel :param consumers: A list of consumers. :type consumers: list """ self.logger.info('Ready to consume') if self.logger.level == logging.DEBUG: queue_names = [] for consumer in consumers: queue_names += [x.name for x in consumer.queues] self.logger.debug( 'Consuming via connection "{0}" and channel "{1}" on ' 'the following queues: "{2}"'.format( connection.as_uri(), channel, '", "'.join(queue_names))) def on_consume_end(self, connection, channel): # pragma: no cover """ Called when the service stops consuming. :param connection: The current connection instance. :type connection: kombu.Connection :param channel: The current channel. :type channel: kombu.transport.*.Channel """ self.logger.warn('Consuming has ended')