def rpc_logging_service(log_level, handlers): """Setup an RPC logger. :param log_level: ``object`` :param handlers: ``object`` """ debug.setup_logging(loglevel=log_level, loggers=handlers)
def __init__(self): config.register_opts(Messaging.amqp_opts) if CONF.debug: setup_logging(loglevel='DEBUG', loggers=['']) self.connection = None self.connect()
def test_adds_handlers_sets_level(self): with patch('kombu.utils.debug.get_logger') as get_logger: logger = get_logger.return_value = Mock() setup_logging(loggers=['kombu.test']) get_logger.assert_called_with('kombu.test') self.assertTrue(logger.addHandler.called) logger.setLevel.assert_called_with(logging.DEBUG)
def run_consumer(): setup_logging(loglevel='INFO') logger.info("Connecting to AMQP ...") with Connection(connection_string) as conn: logger.info("Connected.") logger.info("Awaiting messages ...") try: Worker(conn).run() except KeyboardInterrupt: logger.info("Exiting ...")
def run_consumer(): setup_logging(loglevel='INFO') print "Connecting ..." with Connection(connection_string) as conn: print "Connected." print "Awaiting tasks ..." try: Worker(conn).run() except KeyboardInterrupt: print('kthnxbye')
def disable_debugging(): global ENABLE_DEBUGGING ENABLE_DEBUGGING = False set_log_level_for_all_loggers(level=stdlib_logging.INFO) setup_logging(loglevel=stdlib_logging.INFO) paramiko.common.logging.basicConfig(level=paramiko.common.INFO) return ENABLE_DEBUGGING
def run(self): from sentry.queue.client import broker from sentry.queue.worker import Worker from kombu.utils.debug import setup_logging setup_logging(loglevel="INFO") try: Worker(broker.connection).run() except KeyboardInterrupt: print("bye bye")
def __init__(self, connection, exchange, queue, routing_key, sub_worker, thread_support=True, no_declare=True): """Init a new ConsumerMixin object. Args: connection (kombu.Connection): The Kombu Connection object context. exchange (str): The Exchange to use on RabbitMQ. queue (str): The listen queue to use on RabbitMQ. routing_key (str): The routing key to use on RabbitMQ. sub_worker (str): Name of subworker as a string: ex: `package.module.className`. thread_support (bool, optional): Defaults to True. Does the worker support Thread behavior? no_declare (bool, optional): Defaults to True. Declare Exchange and Queue when connecting? """ # Reduce logging from amqp module setup_logging(loglevel='INFO', loggers=['amqp']) logger.debug( f"Initializating a new listener for exchange/queue: {exchange}/{queue}..." ) self.connection = connection self.exchange = Exchange(exchange, 'direct', durable=True, no_declare=no_declare) self.queue = Queue(queue, exchange=self.exchange, routing_key=routing_key, no_declare=no_declare) self.queues = [self.queue] self.no_declare = no_declare logger.info( f"New listener initialized for exchange/queue: {exchange}/{queue}..." ) logger.debug(f"Importing sub_worker module: {sub_worker}...") self.sub_worker = sub_worker self.thread_support = thread_support mod_name = '.'.join(self.sub_worker.split(".")[:-1]) try: self.sub_worker_mod = importlib.import_module(mod_name) except ModuleNotFoundError as e: logger.error( f"Cannot import the sub worker module named {mod_name}: ModuleNotFoundError" ) sys.exit(-1) except Exception as e: logger.error( f"Cannot import the sub worker module named {mod_name}: " + str(e)) sys.exit(-1)
def monitor(): from kombu import Connection from kombu.utils.debug import setup_logging # setup root logger setup_logging(loglevel='INFO', loggers=['']) with Connection('amqp://*****:*****@localhost:5672//') as conn: try: worker = Worker(conn) worker.run() except KeyboardInterrupt: print('bye bye')
def enable_debugging(): global ENABLE_DEBUGGING ENABLE_DEBUGGING = True # Set debug level for all StackStorm loggers set_log_level_for_all_loggers(level=stdlib_logging.DEBUG) # Set debug log level for kombu setup_logging(loglevel=stdlib_logging.DEBUG) # Set debug log level for paramiko paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG) return ENABLE_DEBUGGING
def __init__(self): if app.debug: setup_logging(loglevel='DEBUG', loggers=['']) self.connection = BrokerConnection(app.config['AMQP_URL']) try: self.connection.connect() except Exception as e: LOG.error('Failed to connect to AMQP transport %s: %s', app.config['AMQP_URL'], e) raise RuntimeError self.channel = self.connection.channel() self.exchange_name = app.config['AMQP_TOPIC'] self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel) self.producer = Producer(exchange=self.exchange, channel=self.channel) LOG.info('Configured fanout publisher on topic "%s"', app.config['AMQP_TOPIC'])
def __init__(self, name=None): if app.config['DEBUG']: setup_logging(loglevel='DEBUG', loggers=['']) self.connection = BrokerConnection(AMQP_URL) try: self.connection.connect() except Exception as e: LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL, e) raise RuntimeError self.channel = self.connection.channel() self.exchange_name = AMQP_TOPIC self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel) self.producer = Producer(exchange=self.exchange, channel=self.channel) super(FanoutPublisher, self).__init__(name) LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
print('Got task: {0!r}'.format(body)) message.ack() # def run_consumer_server(conn, queues): # print "Connecting ..." # with Connection(connection_string) as conn: # # try: # worker = Worker(conn, queues) # worker.run() # except: # raise if __name__ == "__main__": setup_logging(loglevel=logging.DEBUG) conn = Connection(TRANSPORT_URL, heartbeat=4) exchange = Exchange(TRANSPORT_EXCHANGE, type="direct") queues = [ Queue(name=QUEUE_PREFIX + MCOS_CLUSTER_NAME, exchange=exchange, routing_key=CLUSTER_STATUS_ROUTING_KEY), ] for queue in queues: queue.maybe_bind(conn) queue.declare() with conn: print "Connected." print "Awaiting tasks ..." try: worker = Worker(conn, queues)
return ModuleType.__getattribute__(self, name) def __dir__(self): result = list(new_module.__all__) result.extend(("__file__", "__path__", "__doc__", "__all__", "__docformat__", "__name__", "__path__", "VERSION", "__package__", "__version__", "__author__", "__contact__", "__homepage__", "__docformat__")) return result # keep a reference to this module so that it's not garbage collected old_module = sys.modules[__name__] new_module = sys.modules[__name__] = module(__name__) new_module.__dict__.update({ "__file__": __file__, "__path__": __path__, "__doc__": __doc__, "__all__": tuple(object_origins), "__version__": __version__, "__author__": __author__, "__contact__": __contact__, "__homepage__": __homepage__, "__docformat__": __docformat__, "VERSION": VERSION}) if os.environ.get("KOMBU_LOG_DEBUG"): os.environ.update(KOMBU_LOG_CHANNEL="1", KOMBU_LOG_CONNECTION="1") from kombu.utils import debug debug.setup_logging()
__FILENAME__ = test-consumer #!/usr/bin/python from __future__ import with_statement import sys import os import ConfigParser from kombu import BrokerConnection from kombu.utils.debug import setup_logging setup_logging(loglevel="INFO") config_file = '/etc/logix/logix.conf' if os.path.isfile( '/etc/logix/logix.conf' ) else os.getenv('LOGIX_CONF') if not os.path.isfile(config_file): print "Config file %s not found" % config_file sys.exit(1) config = ConfigParser.RawConfigParser() config.read(config_file) with BrokerConnection(config.get('transport', 'url')) as conn: with conn.SimpleQueue(config.get('transport', 'queue'), serializer="json") as queue: message = queue.get(block=True, timeout=10) if message: print message.payload ########NEW FILE########
class Worker(ConsumerMixin): def __init__(self, connection): self.connection = connection def get_consumers(self, Consumer, channel): return [Consumer(queues=task_queues, callbacks=[self.process_task])] def process_task(self, body, message): fun = body['fun'] args = body['args'] kwargs = body['kwargs'] self.info('Got task: %s', reprcall(fun.__name__, args, kwargs)) try: fun(*args, **kwdict(kwargs)) except Exception, exc: self.error('task raised exception: %r', exc) message.ack() if __name__ == '__main__': from kombu import Connection from kombu.utils.debug import setup_logging setup_logging(loglevel='INFO') with Connection('amqp://*****:*****@localhost:5672//') as conn: try: Worker(conn).run() except KeyboardInterrupt: print('bye bye')
with BrokerConnection("amqp://*****:*****@localhost:5672//") as conn: with producers[conn].acquire(block=True) as producer: maybe_declare(announce_exchange, producer.channel) if job.get("command", None) == "flood": job["command"] = "print" for x in range(100): job["sequence"] = x producer.publish(job, serializer="json", routing_key=routing_key) logger.info("PUBLISH: %s route %s" % (job, routing_key)) else: producer.publish(job, serializer="json", routing_key=routing_key) logger.info("PUBLISH: %s route %s" % (job, routing_key)) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) debug.setup_logging(logging.DEBUG) if len(sys.argv) == 1: payload = {"message": "Hello World", "command": "new-slave", "job-id": str(time.time())} publish_job(payload) sys.exit(0) command = "print" if len(sys.argv) > 1: command = sys.argv[1] routing_key = "announce" if len(sys.argv) > 2: routing_key = sys.argv[2] arg = None
class MessageConsumer(ConsumerMixin): queue = Queue('test-queue-2', exchange=exchange, routing_key='foobar2') def __init__(self, connection): self.connection = connection def get_consumers(self, Consumer, channel): consumer = Consumer([self.queue], callbacks=[self.handle_message], accept=['json']) return [consumer] def handle_message(self, body, message): log.debug('\nReceived message "%s"; type: "%s".\n', body, type(body)) message.ack() # Test deleting the queue # self.should_stop = True # bound_queue = self.queue(self.connection) # bound_queue.delete() if __name__ == '__main__': from kombu.utils.debug import setup_logging setup_logging(loglevel='DEBUG', loggers=['']) options = {'project_id': local_settings.PROJECT_ID} with Connection(transport='kombu_transport:Transport', transport_options=options) as connection: MessageConsumer(connection).run()
def setup_loggers(): # Add kombu loggers setup_logging(loglevel=KOMBU_LOG_LEVEL) # Set up the loggers formatters and handlers from the LOGCONFIG dict dictConfig(LOGCONFIG)
queue_opts = {'durable': False, 'auto_delete': True}, exchange_opts = {'delivery_mode' : 1, 'auto_delete' : True, 'durable' : False}) payload = {"Result": DispatchKernels(method, args)} resp_queue.put(payload, serializer='json') resp_queue.close() except: print "Exception caught : %s" % sys.exc_info()[0] return if __name__ == "__main__": from kombu import BrokerConnection from kombu.utils.debug import setup_logging setup_logging(loglevel="DEBUG") configs = cfg.ConfigOpts() options = [ cfg.StrOpt('rabbit_host', default = 'localhost'), cfg.StrOpt('rabbit_password', required = 'true'), cfg.StrOpt('rabbit_user', default = 'guest')] configs.register_opts( options ) configs(sys.argv[1:]) rh = configs.rabbit_host rp = configs.rabbit_password ru = configs.rabbit_user strBroker = "amqp://" + ru + ":" + rp + "@" + rh + ":5672//" retErr = PyOpenCLInterface.Initialize("GPU")
def main(): global OPTIONS CONFIG_SECTION = "alerta-mailer" config_file = os.environ.get("ALERTA_CONF_FILE") or DEFAULT_OPTIONS["config_file"] # Convert default booleans to its string type, otherwise config.getboolean fails defopts = {k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.iteritems()} config = configparser.RawConfigParser(defaults=defopts) try: config.read(os.path.expanduser(config_file)) except Exception as e: LOG.warning("Problem reading configuration file %s - is this an ini file?", config_file) sys.exit(1) if config.has_section(CONFIG_SECTION): from types import NoneType config_getters = { NoneType: config.get, str: config.get, int: config.getint, float: config.getfloat, bool: config.getboolean, list: lambda s, o: [e.strip() for e in config.get(s, o).split(",")], } for opt in DEFAULT_OPTIONS: # Convert the options to the expected type OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](CONFIG_SECTION, opt) else: sys.stderr.write("Alerta configuration section not found in configuration file\n") OPTIONS = defopts.copy() OPTIONS["endpoint"] = os.environ.get("ALERTA_ENDPOINT") or OPTIONS["endpoint"] OPTIONS["key"] = os.environ.get("ALERTA_API_KEY") or OPTIONS["key"] OPTIONS["smtp_password"] = os.environ.get("SMTP_PASSWORD") or OPTIONS["smtp_password"] if os.environ.get("DEBUG"): OPTIONS["debug"] = True try: mailer = MailSender() mailer.start() except (SystemExit, KeyboardInterrupt): sys.exit(0) except Exception as e: print str(e) sys.exit(1) from kombu.utils.debug import setup_logging setup_logging(loglevel="DEBUG" if OPTIONS["debug"] else "INFO", loggers=[""]) with Connection(OPTIONS["amqp_url"]) as conn: try: consumer = FanoutConsumer(connection=conn) consumer.run() except (SystemExit, KeyboardInterrupt): mailer.should_stop = True mailer.join() sys.exit(0) except Exception as e: print str(e) sys.exit(1)
def main(): global OPTIONS CONFIG_SECTION = 'alerta-mailer' config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS[ 'config_file'] # nopep8 # Convert default booleans to its string type, otherwise config.getboolean fails # nopep8 defopts = { k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.items() } # nopep8 config = configparser.RawConfigParser(defaults=defopts) if os.path.exists("{}.d".format(config_file)): config_path = "{}.d".format(config_file) config_list = [] for files in os.walk(config_path): for filename in files[2]: config_list.append("{}/{}".format(config_path, filename)) config_list.append(os.path.expanduser(config_file)) config_file = config_list try: config.read(config_file) except Exception as e: LOG.warning( "Problem reading configuration file %s - is this an ini file?", config_file) # nopep8 sys.exit(1) if config.has_section(CONFIG_SECTION): from types import NoneType config_getters = { NoneType: config.get, str: config.get, int: config.getint, float: config.getfloat, bool: config.getboolean, list: lambda s, o: [e.strip() for e in config.get(s, o).split(',')] } for opt in DEFAULT_OPTIONS: # Convert the options to the expected type OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])]( CONFIG_SECTION, opt) # nopep8 else: sys.stderr.write( 'Alerta configuration section not found in configuration file\n' ) # nopep8 OPTIONS = defopts.copy() OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS[ 'endpoint'] # nopep8 OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key'] OPTIONS['smtp_username'] = os.environ.get( 'SMTP_USERNAME') or OPTIONS['smtp_username'] or OPTIONS['mail_from'] OPTIONS['smtp_password'] = os.environ.get('SMTP_PASSWORD') or OPTIONS[ 'smtp_password'] # nopep8 if os.environ.get('DEBUG'): OPTIONS['debug'] = True group_rules = parse_group_rules(config_file) if group_rules is not None: OPTIONS['group_rules'] = group_rules try: mailer = MailSender() mailer.start() except (SystemExit, KeyboardInterrupt): sys.exit(0) except Exception as e: print(str(e)) sys.exit(1) from kombu.utils.debug import setup_logging loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO' setup_logging(loglevel=loginfo, loggers=['']) with Connection(OPTIONS['amqp_url']) as conn: try: consumer = FanoutConsumer(connection=conn) consumer.run() except (SystemExit, KeyboardInterrupt): mailer.should_stop = True mailer.join() sys.exit(0) except Exception as e: print(str(e)) sys.exit(1)
return [Consumer(queues=[self.task_queue], accept=['json'], callbacks=[self.process_task])] def process_task(self, body, message): for k, v in body.items(): if k == 'payload': print "First message in the payload: ", v[0] print "There are %d events more in this payload..." % (len(v) - 1) else: print "%s = %s" % (k, v) raw_input("Press Enter to continue...") print "\n" message.ack() if __name__ == '__main__': from kombu import Connection from kombu.utils.debug import setup_logging # setup root logger setup_logging(loglevel='DEBUG', loggers=['']) #with Connection('amqp://*****:*****@localhost:5672/', virtual_host='/') as conn: with Connection('amqp://*****:*****@localhost:5672/') as conn: try: print(conn) worker = Worker(conn) worker.run() except KeyboardInterrupt: print('Terminating consumer:bye bye')
def includeme(config): global _producer_connection, _exchange setup_logging(loglevel='DEBUG') url = (config.registry.settings.get('celery_tasks.broker') or config.registry.settings.get('celery_tasks.imap.broker')) _producer_connection = BrokerConnection(url)
print "Returning consumers ..." return [Consumer(queues=task_queues, callbacks=[self.process_task])] def process_task(self, body, message): fun = body['fun'] args = body['args'] kwargs = body['kwargs'] print('Got task: %s on %s' % (reprcall(fun.__name__, args, kwargs), message.delivery_info)) try: fun(*args, **kwdict(kwargs)) except Exception, exc: print('task raised exception: %r' % exc) message.ack() if __name__ == '__main__': from kombu import Connection from kombu.utils.debug import setup_logging setup_logging(loglevel='INFO') with Connection('amqp://*****:*****@localhost:5672//') as conn: try: conn.connect() print "Starting ....", conn.connected Worker(conn).run() except KeyboardInterrupt: print('bye bye')
self.connection = connection def get_consumers(self, Consumer, channel): return [Consumer(queues=task_queues, accept=["pickle", "json"], callbacks=[self.process_task])] def process_task(self, body, message): fun = body["fun"] args = body["args"] kwargs = body["kwargs"] logger.info("Got task: %s", reprcall(fun.__name__, args, kwargs)) try: fun(*args, **kwdict(kwargs)) except Exception as exc: logger.error("task raised exception: %r", exc) message.ack() if __name__ == "__main__": from kombu import Connection from kombu.utils.debug import setup_logging # setup root logger setup_logging(loglevel="INFO", loggers=[""]) with Connection("amqp://*****:*****@localhost:5672//") as conn: try: worker = Worker(conn) worker.run() except KeyboardInterrupt: print("bye bye")
from kombu import Connection, Exchange, Queue, Consumer import socket from datetime import datetime rabbit_url = "amqp://*****:*****@134.221.121.65:5670/vhost1" #proxy from kombu.log import get_logger from kombu.utils.debug import setup_logging logger = get_logger(__name__) setup_logging(loglevel="DEBUG") conn = Connection(rabbit_url, heartbeat=10) exchange_adaas = Exchange("ex-adaas", type="fanout", durable=True) queue_adaas_pri = Queue("q-adaas-pri", exchange=exchange_adaas, durable=True) def process_message(body, message): #print(str(datetime.now()) + " The body is {}".format(body)) msg = str(datetime.now()) + " {}\n".format(body) print(msg) with open(out_file, "a+") as text_file: text_file.write(msg) message.ack() if "EOT" in body: exit() consumer = Consumer(conn, queues=queue_adaas_pri, callbacks=[process_message], accept=["text/plain"]) #consumer.consume()
def process_task(self, body, message): fun = body['fun'] args = body['args'] kwargs = body['kwargs'] logger.info('Got task: %s', reprcall(fun.__name__, args, kwargs)) try: fun(*args, **kwdict(kwargs)) except Exception as exc: logger.error('task raised exception: %r', exc) message.ack() if __name__ == '__main__': from kombu import Connection from kombu.utils.debug import setup_logging # setup root logger setup_logging(loglevel='INFO', loggers=['']) with Connection('amqp://*****:*****@localhost:5672//') as conn: try: worker = Worker(conn) worker.run() except KeyboardInterrupt: print('bye bye') def hello_task(who="world"): print("Hello %s" % (who, )) from kombu.pools import producers
def main(): global OPTIONS CONFIG_SECTION = 'alerta-mailer' config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS['config_file'] # nopep8 # Convert default booleans to its string type, otherwise config.getboolean fails # nopep8 defopts = {k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.items()} # nopep8 config = configparser.RawConfigParser(defaults=defopts) if os.path.exists("{}.d".format(config_file)): config_path = "{}.d".format(config_file) config_list = [] for files in os.walk(config_path): for filename in files[2]: config_list.append("{}/{}".format(config_path, filename)) config_list.append(os.path.expanduser(config_file)) config_file = config_list try: config.read(os.path.expanduser(config_file)) except Exception as e: LOG.warning("Problem reading configuration file %s - is this an ini file?", config_file) # nopep8 sys.exit(1) if config.has_section(CONFIG_SECTION): NoneType = type(None) config_getters = { NoneType: config.get, str: config.get, int: config.getint, float: config.getfloat, bool: config.getboolean, list: lambda s, o: [e.strip() for e in config.get(s, o).split(',')] } for opt in DEFAULT_OPTIONS: # Convert the options to the expected type OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](CONFIG_SECTION, opt) # nopep8 else: sys.stderr.write('Alerta configuration section not found in configuration file\n') # nopep8 OPTIONS = defopts.copy() OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS['endpoint'] # nopep8 OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key'] OPTIONS['smtp_username'] = os.environ.get('SMTP_USERNAME') or OPTIONS['smtp_username'] or OPTIONS['mail_from'] OPTIONS['smtp_password'] = os.environ.get('SMTP_PASSWORD') or OPTIONS['smtp_password'] # nopep8 if os.environ.get('DEBUG'): OPTIONS['debug'] = True group_rules = parse_group_rules(config_file) if group_rules is not None: OPTIONS['group_rules'] = group_rules try: mailer = MailSender() mailer.start() except (SystemExit, KeyboardInterrupt): sys.exit(0) except Exception as e: print(str(e)) sys.exit(1) from kombu.utils.debug import setup_logging loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO' setup_logging(loglevel=loginfo, loggers=['']) with Connection(OPTIONS['amqp_url']) as conn: try: consumer = FanoutConsumer(connection=conn) consumer.run() except (SystemExit, KeyboardInterrupt): mailer.should_stop = True mailer.join() sys.exit(0) except Exception as e: print(str(e)) sys.exit(1)
#Now Lets declare the task queue where we will pull work messages hello_queue = Queue('Hello', exchange=default_exchange, routing_key='Hello') def __init__(self, connection): self.connection = connection def get_consumers(self, Consumer, channel): return [ Consumer(queues=self.hello_queue, callbacks=[self.process_task], no_ack=True) ] def process_task(self, body, message): print " [x] Received %r" % (body, ) if __name__ == '__main__': from kombu import Connection from kombu.utils.debug import setup_logging # setup root logger setup_logging(loglevel='INFO', loggers=['']) with Connection('amqp://*****:*****@localhost//') as conn: try: worker = BasicReceiver(conn) print '[*] Waiting for messages. To exit press CTRL+C' worker.run() except KeyboardInterrupt: print('Connection Terminated!')
return reader def stop(self): for reader in self.readers.itervalues(): reader.stop() def includeme(config): global _producer_connection, _exchange setup_logging(loglevel='DEBUG') url = config.registry.settings.get('celery_tasks.imap.broker') _producer_connection = BrokerConnection(url) if __name__ == '__main__': if len(sys.argv) != 2: print "usage: python source_reader.py configuration.ini" setup_logging(loglevel='DEBUG') settings = get_appsettings(sys.argv[-1], 'assembl') registry = getGlobalSiteManager() registry.settings = settings set_config(settings) configure(registry, 'source_reader') url = settings.get('celery_tasks.imap.broker') with BrokerConnection(url) as conn: sourcedispatcher = SourceDispatcher(conn) try: sourcedispatcher.run() except KeyboardInterrupt: sourcedispatcher.stop()
def main(): global OPTIONS MAIL_CONFIG_SECTION = 'mail' TELEGRAM_CONFIG_SECTION = 'telegram' config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS[ 'config_file'] # nopep8 # Convert default booleans to its string type, otherwise config.getboolean fails # nopep8 defopts = { k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.items() } # nopep8 config = configparser.RawConfigParser(defaults=defopts) if os.path.exists("{}.d".format(config_file)): config_path = "{}.d".format(config_file) config_list = [] for files in os.walk(config_path): for filename in files[2]: config_list.append("{}/{}".format(config_path, filename)) config_list.append(os.path.expanduser(config_file)) config_file = config_list try: # No need to expanduser if we got a list (already done sooner) # Morever expanduser does not accept a list. if isinstance(config_file, list): config.read(config_file) else: config.read(os.path.expanduser(config_file)) except Exception as e: LOG.warning( "Problem reading configuration file %s - is this an ini file?", config_file) # nopep8 sys.exit(1) NoneType = type(None) config_getters = { NoneType: config.get, str: config.get, int: config.getint, float: config.getfloat, bool: config.getboolean, list: lambda s, o: [e.strip() for e in config.get(s, o).split(',')] if len(config.get(s, o)) else [] } if config.has_section(MAIL_CONFIG_SECTION): for opt in DEFAULT_MAIL_OPTIONS: if config.has_option(MAIL_CONFIG_SECTION, opt): # Convert the options to the expected type MAIL_OPTIONS[opt] = config_getters[type( DEFAULT_MAIL_OPTIONS[opt])](MAIL_CONFIG_SECTION, opt) # nopep8 else: MAIL_OPTIONS[opt] = DEFAULT_MAIL_OPTIONS[opt] else: LOG.debug( 'Alerta mail configuration section not found in configuration file\n' ) # nopep8 # OPTIONS = defopts.copy() if config.has_section(TELEGRAM_CONFIG_SECTION): for opt in DEFAULT_TELEGRAM_OPTIONS: # Convert the options to the expected type if config.has_option(TELEGRAM_CONFIG_SECTION, opt): TELEGRAM_OPTIONS[opt] = config_getters[type( DEFAULT_TELEGRAM_OPTIONS[opt])](TELEGRAM_CONFIG_SECTION, opt) # nopep8 else: TELEGRAM_OPTIONS[opt] = DEFAULT_TELEGRAM_OPTIONS[opt] else: LOG.debug( 'Alerta telegram configuration section not found in configuration file\n' ) # nopep8 # OPTIONS = defopts.copy() for opt in DEFAULT_OPTIONS: # Convert the options to the expected type if config.has_option("DEFAULT", opt): OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])]( "DEFAULT", opt) # nopep8 else: OPTIONS[opt] = DEFAULT_OPTIONS[opt] OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS[ 'endpoint'] # nopep8 OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key'] if isinstance(config_file, list): group_rules = [] for file in config_file: group_rules.extend(parse_group_rules(file)) else: group_rules = parse_group_rules(config_file) if group_rules is not None: OPTIONS['group_rules'] = group_rules # Registering action for SIGTERM signal handling signal.signal(signal.SIGTERM, on_sigterm) try: trigger = Trigger() trigger.start() except (SystemExit, KeyboardInterrupt): sys.exit(0) except Exception as e: print(str(e)) sys.exit(1) from kombu.utils.debug import setup_logging loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO' setup_logging(loglevel=loginfo, loggers=['']) with Connection(OPTIONS['amqp_url']) as conn: try: consumer = FanoutConsumer(connection=conn, hold_time=OPTIONS['hold_time']) consumer.run() except (SystemExit, KeyboardInterrupt): trigger.should_stop = True trigger.join() sys.exit(0) except Exception as e: print(str(e)) sys.exit(1)
from __future__ import with_statement from kombu.mixins import ConsumerMixin from kombu.utils import kwdict from queues import task_queues class Worker(ConsumerMixin): def __init__(self, connection): self.connection = connection def get_consumers(self, Consumer, channel): return [Consumer(queues=task_queues, callbacks=[self.process_task])] def process_task(self, body, message): fun = body["fun"] args = body["args"] kwargs = body["kwargs"] fun(*args, **kwdict(kwargs)) message.ack() if __name__ == "__main__": from kombu import BrokerConnection from kombu.utils.debug import setup_logging setup_logging(loglevel="INFO") with BrokerConnection("amqp://*****:*****@localhost:5672//") as conn: Worker(conn).run()
import json import time from flask import current_app from kombu.mixins import ConsumerMixin import ujson from listenbrainz.listen import Listen, NowPlayingListen from listenbrainz.utils import get_fallback_connection_name from kombu import Connection, Exchange, Queue, Consumer from kombu.utils.debug import setup_logging setup_logging() class ListensDispatcher(ConsumerMixin): def __init__(self, app, socketio): self.app = app self.socketio = socketio self.connection = None # there are two consumers, so we need two channels: one for playing now queue and another # for normal listens queue. when using ConsumerMixin, it sets up a default channel itself. # we create the other channel here. we also need to handle its cleanup later self.playing_now_channel = None self.unique_exchange = Exchange(app.config["UNIQUE_EXCHANGE"], "fanout", durable=False) self.playing_now_exchange = Exchange(
parser.add_argument( '--interval', '-i', type=int, default=30, help='Interval to check reddit at' ) parser.add_argument( '--config', '-c', type=str, default='/etc/redditbot.yml', help='Path to YAML config file' ) parser.add_argument( '--log', '-l', type=str, default='info', choices=('debug', 'info', 'warning'), help='Logging Level' ) args = parser.parse_args() setup_logging(loglevel=args.log.upper(), loggers=['']) loop = asyncio.get_event_loop() tasks = [] with open(args.config, 'r') as ymlfile: redditbot.config = yaml.load(ymlfile) if args.client is True: from redditbot.client import UpdateSubreddit for subreddit in redditbot.config['subreddits'].keys(): tasks.append(UpdateSubreddit(subreddit, args.interval)) if args.worker is True: from redditbot import bot # noqa from redditbot import worker