def rpc_logging_service(log_level, handlers):
"""Setup an RPC logger.
:param log_level: ``object``
:param handlers: ``object``
"""
debug.setup_logging(loglevel=log_level, loggers=handlers)
def rpc_logging_service(log_level, handlers):
"""Setup an RPC logger.
:param log_level: ``object``
:param handlers: ``object``
"""
debug.setup_logging(loglevel=log_level, loggers=handlers)
def __init__(self):
config.register_opts(Messaging.amqp_opts)
if CONF.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = None
self.connect()
def test_adds_handlers_sets_level(self):
with patch('kombu.utils.debug.get_logger') as get_logger:
logger = get_logger.return_value = Mock()
setup_logging(loggers=['kombu.test'])
get_logger.assert_called_with('kombu.test')
self.assertTrue(logger.addHandler.called)
logger.setLevel.assert_called_with(logging.DEBUG)
def test_adds_handlers_sets_level(self):
with patch('kombu.utils.debug.get_logger') as get_logger:
logger = get_logger.return_value = Mock()
setup_logging(loggers=['kombu.test'])
get_logger.assert_called_with('kombu.test')
self.assertTrue(logger.addHandler.called)
logger.setLevel.assert_called_with(logging.DEBUG)
def run_consumer():
setup_logging(loglevel='INFO')
logger.info("Connecting to AMQP ...")
with Connection(connection_string) as conn:
logger.info("Connected.")
logger.info("Awaiting messages ...")
try:
Worker(conn).run()
except KeyboardInterrupt:
logger.info("Exiting ...")
def run_consumer():
setup_logging(loglevel='INFO')
print "Connecting ..."
with Connection(connection_string) as conn:
print "Connected."
print "Awaiting tasks ..."
try:
Worker(conn).run()
except KeyboardInterrupt:
print('kthnxbye')
def disable_debugging():
global ENABLE_DEBUGGING
ENABLE_DEBUGGING = False
set_log_level_for_all_loggers(level=stdlib_logging.INFO)
setup_logging(loglevel=stdlib_logging.INFO)
paramiko.common.logging.basicConfig(level=paramiko.common.INFO)
return ENABLE_DEBUGGING
def disable_debugging():
global ENABLE_DEBUGGING
ENABLE_DEBUGGING = False
set_log_level_for_all_loggers(level=stdlib_logging.INFO)
setup_logging(loglevel=stdlib_logging.INFO)
paramiko.common.logging.basicConfig(level=paramiko.common.INFO)
return ENABLE_DEBUGGING
def run(self):
from sentry.queue.client import broker
from sentry.queue.worker import Worker
from kombu.utils.debug import setup_logging
setup_logging(loglevel="INFO")
try:
Worker(broker.connection).run()
except KeyboardInterrupt:
print("bye bye")
def __init__(self,
connection,
exchange,
queue,
routing_key,
sub_worker,
thread_support=True,
no_declare=True):
"""Init a new ConsumerMixin object.
Args:
connection (kombu.Connection): The Kombu Connection object context.
exchange (str): The Exchange to use on RabbitMQ.
queue (str): The listen queue to use on RabbitMQ.
routing_key (str): The routing key to use on RabbitMQ.
sub_worker (str): Name of subworker as a string: ex: `package.module.className`.
thread_support (bool, optional): Defaults to True. Does the worker support Thread behavior?
no_declare (bool, optional): Defaults to True. Declare Exchange and Queue when connecting?
"""
# Reduce logging from amqp module
setup_logging(loglevel='INFO', loggers=['amqp'])
logger.debug(
f"Initializating a new listener for exchange/queue: {exchange}/{queue}..."
)
self.connection = connection
self.exchange = Exchange(exchange,
'direct',
durable=True,
no_declare=no_declare)
self.queue = Queue(queue,
exchange=self.exchange,
routing_key=routing_key,
no_declare=no_declare)
self.queues = [self.queue]
self.no_declare = no_declare
logger.info(
f"New listener initialized for exchange/queue: {exchange}/{queue}..."
)
logger.debug(f"Importing sub_worker module: {sub_worker}...")
self.sub_worker = sub_worker
self.thread_support = thread_support
mod_name = '.'.join(self.sub_worker.split(".")[:-1])
try:
self.sub_worker_mod = importlib.import_module(mod_name)
except ModuleNotFoundError as e:
logger.error(
f"Cannot import the sub worker module named {mod_name}: ModuleNotFoundError"
)
sys.exit(-1)
except Exception as e:
logger.error(
f"Cannot import the sub worker module named {mod_name}: " +
str(e))
sys.exit(-1)
def monitor():
from kombu import Connection
from kombu.utils.debug import setup_logging
# setup root logger
setup_logging(loglevel='INFO', loggers=[''])
with Connection('amqp://*****:*****@localhost:5672//') as conn:
try:
worker = Worker(conn)
worker.run()
except KeyboardInterrupt:
print('bye bye')
def enable_debugging():
global ENABLE_DEBUGGING
ENABLE_DEBUGGING = True
# Set debug level for all StackStorm loggers
set_log_level_for_all_loggers(level=stdlib_logging.DEBUG)
# Set debug log level for kombu
setup_logging(loglevel=stdlib_logging.DEBUG)
# Set debug log level for paramiko
paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG)
return ENABLE_DEBUGGING
def enable_debugging():
global ENABLE_DEBUGGING
ENABLE_DEBUGGING = True
# Set debug level for all StackStorm loggers
set_log_level_for_all_loggers(level=stdlib_logging.DEBUG)
# Set debug log level for kombu
setup_logging(loglevel=stdlib_logging.DEBUG)
# Set debug log level for paramiko
paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG)
return ENABLE_DEBUGGING
def __init__(self):
if app.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(app.config['AMQP_URL'])
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', app.config['AMQP_URL'], e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = app.config['AMQP_TOPIC']
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
LOG.info('Configured fanout publisher on topic "%s"', app.config['AMQP_TOPIC'])
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL, e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
print('Got task: {0!r}'.format(body))
message.ack()
# def run_consumer_server(conn, queues):
# print "Connecting ..."
# with Connection(connection_string) as conn:
#
# try:
# worker = Worker(conn, queues)
# worker.run()
# except:
# raise
if __name__ == "__main__":
setup_logging(loglevel=logging.DEBUG)
conn = Connection(TRANSPORT_URL, heartbeat=4)
exchange = Exchange(TRANSPORT_EXCHANGE, type="direct")
queues = [
Queue(name=QUEUE_PREFIX + MCOS_CLUSTER_NAME,
exchange=exchange,
routing_key=CLUSTER_STATUS_ROUTING_KEY),
]
for queue in queues:
queue.maybe_bind(conn)
queue.declare()
with conn:
print "Connected."
print "Awaiting tasks ..."
try:
worker = Worker(conn, queues)
return ModuleType.__getattribute__(self, name)
def __dir__(self):
result = list(new_module.__all__)
result.extend(("__file__", "__path__", "__doc__", "__all__",
"__docformat__", "__name__", "__path__", "VERSION",
"__package__", "__version__", "__author__",
"__contact__", "__homepage__", "__docformat__"))
return result
# keep a reference to this module so that it's not garbage collected
old_module = sys.modules[__name__]
new_module = sys.modules[__name__] = module(__name__)
new_module.__dict__.update({
"__file__": __file__,
"__path__": __path__,
"__doc__": __doc__,
"__all__": tuple(object_origins),
"__version__": __version__,
"__author__": __author__,
"__contact__": __contact__,
"__homepage__": __homepage__,
"__docformat__": __docformat__,
"VERSION": VERSION})
if os.environ.get("KOMBU_LOG_DEBUG"):
os.environ.update(KOMBU_LOG_CHANNEL="1", KOMBU_LOG_CONNECTION="1")
from kombu.utils import debug
debug.setup_logging()
__FILENAME__ = test-consumer
#!/usr/bin/python
from __future__ import with_statement
import sys
import os
import ConfigParser
from kombu import BrokerConnection
from kombu.utils.debug import setup_logging
setup_logging(loglevel="INFO")
config_file = '/etc/logix/logix.conf' if os.path.isfile(
'/etc/logix/logix.conf'
) else os.getenv('LOGIX_CONF')
if not os.path.isfile(config_file):
print "Config file %s not found" % config_file
sys.exit(1)
config = ConfigParser.RawConfigParser()
config.read(config_file)
with BrokerConnection(config.get('transport', 'url')) as conn:
with conn.SimpleQueue(config.get('transport', 'queue'),
serializer="json") as queue:
message = queue.get(block=True, timeout=10)
if message:
print message.payload
########NEW FILE########
return ModuleType.__getattribute__(self, name)
def __dir__(self):
result = list(new_module.__all__)
result.extend(("__file__", "__path__", "__doc__", "__all__",
"__docformat__", "__name__", "__path__", "VERSION",
"__package__", "__version__", "__author__",
"__contact__", "__homepage__", "__docformat__"))
return result
# keep a reference to this module so that it's not garbage collected
old_module = sys.modules[__name__]
new_module = sys.modules[__name__] = module(__name__)
new_module.__dict__.update({
"__file__": __file__,
"__path__": __path__,
"__doc__": __doc__,
"__all__": tuple(object_origins),
"__version__": __version__,
"__author__": __author__,
"__contact__": __contact__,
"__homepage__": __homepage__,
"__docformat__": __docformat__,
"VERSION": VERSION})
if os.environ.get("KOMBU_LOG_DEBUG"):
os.environ.update(KOMBU_LOG_CHANNEL="1", KOMBU_LOG_CONNECTION="1")
from kombu.utils import debug
debug.setup_logging()
class Worker(ConsumerMixin):
def __init__(self, connection):
self.connection = connection
def get_consumers(self, Consumer, channel):
return [Consumer(queues=task_queues, callbacks=[self.process_task])]
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
self.info('Got task: %s', reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
self.error('task raised exception: %r', exc)
message.ack()
if __name__ == '__main__':
from kombu import Connection
from kombu.utils.debug import setup_logging
setup_logging(loglevel='INFO')
with Connection('amqp://*****:*****@localhost:5672//') as conn:
try:
Worker(conn).run()
except KeyboardInterrupt:
print('bye bye')
with BrokerConnection("amqp://*****:*****@localhost:5672//") as conn:
with producers[conn].acquire(block=True) as producer:
maybe_declare(announce_exchange, producer.channel)
if job.get("command", None) == "flood":
job["command"] = "print"
for x in range(100):
job["sequence"] = x
producer.publish(job, serializer="json", routing_key=routing_key)
logger.info("PUBLISH: %s route %s" % (job, routing_key))
else:
producer.publish(job, serializer="json", routing_key=routing_key)
logger.info("PUBLISH: %s route %s" % (job, routing_key))
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
debug.setup_logging(logging.DEBUG)
if len(sys.argv) == 1:
payload = {"message": "Hello World", "command": "new-slave", "job-id": str(time.time())}
publish_job(payload)
sys.exit(0)
command = "print"
if len(sys.argv) > 1:
command = sys.argv[1]
routing_key = "announce"
if len(sys.argv) > 2:
routing_key = sys.argv[2]
arg = None
class MessageConsumer(ConsumerMixin):
queue = Queue('test-queue-2', exchange=exchange, routing_key='foobar2')
def __init__(self, connection):
self.connection = connection
def get_consumers(self, Consumer, channel):
consumer = Consumer([self.queue],
callbacks=[self.handle_message],
accept=['json'])
return [consumer]
def handle_message(self, body, message):
log.debug('\nReceived message "%s"; type: "%s".\n', body, type(body))
message.ack()
# Test deleting the queue
# self.should_stop = True
# bound_queue = self.queue(self.connection)
# bound_queue.delete()
if __name__ == '__main__':
from kombu.utils.debug import setup_logging
setup_logging(loglevel='DEBUG', loggers=[''])
options = {'project_id': local_settings.PROJECT_ID}
with Connection(transport='kombu_transport:Transport',
transport_options=options) as connection:
MessageConsumer(connection).run()
def setup_loggers():
# Add kombu loggers
setup_logging(loglevel=KOMBU_LOG_LEVEL)
# Set up the loggers formatters and handlers from the LOGCONFIG dict
dictConfig(LOGCONFIG)
queue_opts = {'durable': False, 'auto_delete': True},
exchange_opts = {'delivery_mode' : 1,
'auto_delete' : True,
'durable' : False})
payload = {"Result": DispatchKernels(method, args)}
resp_queue.put(payload, serializer='json')
resp_queue.close()
except:
print "Exception caught : %s" % sys.exc_info()[0]
return
if __name__ == "__main__":
from kombu import BrokerConnection
from kombu.utils.debug import setup_logging
setup_logging(loglevel="DEBUG")
configs = cfg.ConfigOpts()
options = [ cfg.StrOpt('rabbit_host', default = 'localhost'),
cfg.StrOpt('rabbit_password', required = 'true'),
cfg.StrOpt('rabbit_user', default = 'guest')]
configs.register_opts( options )
configs(sys.argv[1:])
rh = configs.rabbit_host
rp = configs.rabbit_password
ru = configs.rabbit_user
strBroker = "amqp://" + ru + ":" + rp + "@" + rh + ":5672//"
retErr = PyOpenCLInterface.Initialize("GPU")
def main():
global OPTIONS
CONFIG_SECTION = "alerta-mailer"
config_file = os.environ.get("ALERTA_CONF_FILE") or DEFAULT_OPTIONS["config_file"]
# Convert default booleans to its string type, otherwise config.getboolean fails
defopts = {k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.iteritems()}
config = configparser.RawConfigParser(defaults=defopts)
try:
config.read(os.path.expanduser(config_file))
except Exception as e:
LOG.warning("Problem reading configuration file %s - is this an ini file?", config_file)
sys.exit(1)
if config.has_section(CONFIG_SECTION):
from types import NoneType
config_getters = {
NoneType: config.get,
str: config.get,
int: config.getint,
float: config.getfloat,
bool: config.getboolean,
list: lambda s, o: [e.strip() for e in config.get(s, o).split(",")],
}
for opt in DEFAULT_OPTIONS:
# Convert the options to the expected type
OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](CONFIG_SECTION, opt)
else:
sys.stderr.write("Alerta configuration section not found in configuration file\n")
OPTIONS = defopts.copy()
OPTIONS["endpoint"] = os.environ.get("ALERTA_ENDPOINT") or OPTIONS["endpoint"]
OPTIONS["key"] = os.environ.get("ALERTA_API_KEY") or OPTIONS["key"]
OPTIONS["smtp_password"] = os.environ.get("SMTP_PASSWORD") or OPTIONS["smtp_password"]
if os.environ.get("DEBUG"):
OPTIONS["debug"] = True
try:
mailer = MailSender()
mailer.start()
except (SystemExit, KeyboardInterrupt):
sys.exit(0)
except Exception as e:
print str(e)
sys.exit(1)
from kombu.utils.debug import setup_logging
setup_logging(loglevel="DEBUG" if OPTIONS["debug"] else "INFO", loggers=[""])
with Connection(OPTIONS["amqp_url"]) as conn:
try:
consumer = FanoutConsumer(connection=conn)
consumer.run()
except (SystemExit, KeyboardInterrupt):
mailer.should_stop = True
mailer.join()
sys.exit(0)
except Exception as e:
print str(e)
sys.exit(1)
def main():
global OPTIONS
CONFIG_SECTION = 'alerta-mailer'
config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS[
'config_file'] # nopep8
# Convert default booleans to its string type, otherwise config.getboolean fails # nopep8
defopts = {
k: str(v) if type(v) is bool else v
for k, v in DEFAULT_OPTIONS.items()
} # nopep8
config = configparser.RawConfigParser(defaults=defopts)
if os.path.exists("{}.d".format(config_file)):
config_path = "{}.d".format(config_file)
config_list = []
for files in os.walk(config_path):
for filename in files[2]:
config_list.append("{}/{}".format(config_path, filename))
config_list.append(os.path.expanduser(config_file))
config_file = config_list
try:
config.read(config_file)
except Exception as e:
LOG.warning(
"Problem reading configuration file %s - is this an ini file?",
config_file) # nopep8
sys.exit(1)
if config.has_section(CONFIG_SECTION):
from types import NoneType
config_getters = {
NoneType: config.get,
str: config.get,
int: config.getint,
float: config.getfloat,
bool: config.getboolean,
list:
lambda s, o: [e.strip() for e in config.get(s, o).split(',')]
}
for opt in DEFAULT_OPTIONS:
# Convert the options to the expected type
OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](
CONFIG_SECTION, opt) # nopep8
else:
sys.stderr.write(
'Alerta configuration section not found in configuration file\n'
) # nopep8
OPTIONS = defopts.copy()
OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS[
'endpoint'] # nopep8
OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key']
OPTIONS['smtp_username'] = os.environ.get(
'SMTP_USERNAME') or OPTIONS['smtp_username'] or OPTIONS['mail_from']
OPTIONS['smtp_password'] = os.environ.get('SMTP_PASSWORD') or OPTIONS[
'smtp_password'] # nopep8
if os.environ.get('DEBUG'):
OPTIONS['debug'] = True
group_rules = parse_group_rules(config_file)
if group_rules is not None:
OPTIONS['group_rules'] = group_rules
try:
mailer = MailSender()
mailer.start()
except (SystemExit, KeyboardInterrupt):
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
from kombu.utils.debug import setup_logging
loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO'
setup_logging(loglevel=loginfo, loggers=[''])
with Connection(OPTIONS['amqp_url']) as conn:
try:
consumer = FanoutConsumer(connection=conn)
consumer.run()
except (SystemExit, KeyboardInterrupt):
mailer.should_stop = True
mailer.join()
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
return [Consumer(queues=[self.task_queue],
accept=['json'],
callbacks=[self.process_task])]
def process_task(self, body, message):
for k, v in body.items():
if k == 'payload':
print "First message in the payload: ", v[0]
print "There are %d events more in this payload..." % (len(v) - 1)
else:
print "%s = %s" % (k, v)
raw_input("Press Enter to continue...")
print "\n"
message.ack()
if __name__ == '__main__':
from kombu import Connection
from kombu.utils.debug import setup_logging
# setup root logger
setup_logging(loglevel='DEBUG', loggers=[''])
#with Connection('amqp://*****:*****@localhost:5672/', virtual_host='/') as conn:
with Connection('amqp://*****:*****@localhost:5672/') as conn:
try:
print(conn)
worker = Worker(conn)
worker.run()
except KeyboardInterrupt:
print('Terminating consumer:bye bye')
def includeme(config):
global _producer_connection, _exchange
setup_logging(loglevel='DEBUG')
url = (config.registry.settings.get('celery_tasks.broker') or
config.registry.settings.get('celery_tasks.imap.broker'))
_producer_connection = BrokerConnection(url)
print "Returning consumers ..."
return [Consumer(queues=task_queues,
callbacks=[self.process_task])]
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
print('Got task: %s on %s' %
(reprcall(fun.__name__, args, kwargs),
message.delivery_info))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
print('task raised exception: %r' % exc)
message.ack()
if __name__ == '__main__':
from kombu import Connection
from kombu.utils.debug import setup_logging
setup_logging(loglevel='INFO')
with Connection('amqp://*****:*****@localhost:5672//') as conn:
try:
conn.connect()
print "Starting ....", conn.connected
Worker(conn).run()
except KeyboardInterrupt:
print('bye bye')
self.connection = connection
def get_consumers(self, Consumer, channel):
return [Consumer(queues=task_queues, accept=["pickle", "json"], callbacks=[self.process_task])]
def process_task(self, body, message):
fun = body["fun"]
args = body["args"]
kwargs = body["kwargs"]
logger.info("Got task: %s", reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception as exc:
logger.error("task raised exception: %r", exc)
message.ack()
if __name__ == "__main__":
from kombu import Connection
from kombu.utils.debug import setup_logging
# setup root logger
setup_logging(loglevel="INFO", loggers=[""])
with Connection("amqp://*****:*****@localhost:5672//") as conn:
try:
worker = Worker(conn)
worker.run()
except KeyboardInterrupt:
print("bye bye")
from kombu import Connection, Exchange, Queue, Consumer
import socket
from datetime import datetime
rabbit_url = "amqp://*****:*****@134.221.121.65:5670/vhost1" #proxy
from kombu.log import get_logger
from kombu.utils.debug import setup_logging
logger = get_logger(__name__)
setup_logging(loglevel="DEBUG")
conn = Connection(rabbit_url, heartbeat=10)
exchange_adaas = Exchange("ex-adaas", type="fanout", durable=True)
queue_adaas_pri = Queue("q-adaas-pri", exchange=exchange_adaas, durable=True)
def process_message(body, message):
#print(str(datetime.now()) + " The body is {}".format(body))
msg = str(datetime.now()) + " {}\n".format(body)
print(msg)
with open(out_file, "a+") as text_file:
text_file.write(msg)
message.ack()
if "EOT" in body:
exit()
consumer = Consumer(conn,
queues=queue_adaas_pri,
callbacks=[process_message],
accept=["text/plain"])
#consumer.consume()
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
logger.info('Got task: %s', reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception as exc:
logger.error('task raised exception: %r', exc)
message.ack()
if __name__ == '__main__':
from kombu import Connection
from kombu.utils.debug import setup_logging
# setup root logger
setup_logging(loglevel='INFO', loggers=[''])
with Connection('amqp://*****:*****@localhost:5672//') as conn:
try:
worker = Worker(conn)
worker.run()
except KeyboardInterrupt:
print('bye bye')
def hello_task(who="world"):
print("Hello %s" % (who, ))
from kombu.pools import producers
def main():
global OPTIONS
CONFIG_SECTION = 'alerta-mailer'
config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS['config_file'] # nopep8
# Convert default booleans to its string type, otherwise config.getboolean fails # nopep8
defopts = {k: str(v) if type(v) is bool else v for k, v in DEFAULT_OPTIONS.items()} # nopep8
config = configparser.RawConfigParser(defaults=defopts)
if os.path.exists("{}.d".format(config_file)):
config_path = "{}.d".format(config_file)
config_list = []
for files in os.walk(config_path):
for filename in files[2]:
config_list.append("{}/{}".format(config_path, filename))
config_list.append(os.path.expanduser(config_file))
config_file = config_list
try:
config.read(os.path.expanduser(config_file))
except Exception as e:
LOG.warning("Problem reading configuration file %s - is this an ini file?", config_file) # nopep8
sys.exit(1)
if config.has_section(CONFIG_SECTION):
NoneType = type(None)
config_getters = {
NoneType: config.get,
str: config.get,
int: config.getint,
float: config.getfloat,
bool: config.getboolean,
list: lambda s, o: [e.strip() for e in config.get(s, o).split(',')]
}
for opt in DEFAULT_OPTIONS:
# Convert the options to the expected type
OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](CONFIG_SECTION, opt) # nopep8
else:
sys.stderr.write('Alerta configuration section not found in configuration file\n') # nopep8
OPTIONS = defopts.copy()
OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS['endpoint'] # nopep8
OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key']
OPTIONS['smtp_username'] = os.environ.get('SMTP_USERNAME') or OPTIONS['smtp_username'] or OPTIONS['mail_from']
OPTIONS['smtp_password'] = os.environ.get('SMTP_PASSWORD') or OPTIONS['smtp_password'] # nopep8
if os.environ.get('DEBUG'):
OPTIONS['debug'] = True
group_rules = parse_group_rules(config_file)
if group_rules is not None:
OPTIONS['group_rules'] = group_rules
try:
mailer = MailSender()
mailer.start()
except (SystemExit, KeyboardInterrupt):
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
from kombu.utils.debug import setup_logging
loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO'
setup_logging(loglevel=loginfo, loggers=[''])
with Connection(OPTIONS['amqp_url']) as conn:
try:
consumer = FanoutConsumer(connection=conn)
consumer.run()
except (SystemExit, KeyboardInterrupt):
mailer.should_stop = True
mailer.join()
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
#Now Lets declare the task queue where we will pull work messages
hello_queue = Queue('Hello',
exchange=default_exchange,
routing_key='Hello')
def __init__(self, connection):
self.connection = connection
def get_consumers(self, Consumer, channel):
return [
Consumer(queues=self.hello_queue,
callbacks=[self.process_task],
no_ack=True)
]
def process_task(self, body, message):
print " [x] Received %r" % (body, )
if __name__ == '__main__':
from kombu import Connection
from kombu.utils.debug import setup_logging
# setup root logger
setup_logging(loglevel='INFO', loggers=[''])
with Connection('amqp://*****:*****@localhost//') as conn:
try:
worker = BasicReceiver(conn)
print '[*] Waiting for messages. To exit press CTRL+C'
worker.run()
except KeyboardInterrupt:
print('Connection Terminated!')
return reader
def stop(self):
for reader in self.readers.itervalues():
reader.stop()
def includeme(config):
global _producer_connection, _exchange
setup_logging(loglevel='DEBUG')
url = config.registry.settings.get('celery_tasks.imap.broker')
_producer_connection = BrokerConnection(url)
if __name__ == '__main__':
if len(sys.argv) != 2:
print "usage: python source_reader.py configuration.ini"
setup_logging(loglevel='DEBUG')
settings = get_appsettings(sys.argv[-1], 'assembl')
registry = getGlobalSiteManager()
registry.settings = settings
set_config(settings)
configure(registry, 'source_reader')
url = settings.get('celery_tasks.imap.broker')
with BrokerConnection(url) as conn:
sourcedispatcher = SourceDispatcher(conn)
try:
sourcedispatcher.run()
except KeyboardInterrupt:
sourcedispatcher.stop()
def main():
global OPTIONS
MAIL_CONFIG_SECTION = 'mail'
TELEGRAM_CONFIG_SECTION = 'telegram'
config_file = os.environ.get('ALERTA_CONF_FILE') or DEFAULT_OPTIONS[
'config_file'] # nopep8
# Convert default booleans to its string type, otherwise config.getboolean fails # nopep8
defopts = {
k: str(v) if type(v) is bool else v
for k, v in DEFAULT_OPTIONS.items()
} # nopep8
config = configparser.RawConfigParser(defaults=defopts)
if os.path.exists("{}.d".format(config_file)):
config_path = "{}.d".format(config_file)
config_list = []
for files in os.walk(config_path):
for filename in files[2]:
config_list.append("{}/{}".format(config_path, filename))
config_list.append(os.path.expanduser(config_file))
config_file = config_list
try:
# No need to expanduser if we got a list (already done sooner)
# Morever expanduser does not accept a list.
if isinstance(config_file, list):
config.read(config_file)
else:
config.read(os.path.expanduser(config_file))
except Exception as e:
LOG.warning(
"Problem reading configuration file %s - is this an ini file?",
config_file) # nopep8
sys.exit(1)
NoneType = type(None)
config_getters = {
NoneType:
config.get,
str:
config.get,
int:
config.getint,
float:
config.getfloat,
bool:
config.getboolean,
list:
lambda s, o: [e.strip() for e in config.get(s, o).split(',')]
if len(config.get(s, o)) else []
}
if config.has_section(MAIL_CONFIG_SECTION):
for opt in DEFAULT_MAIL_OPTIONS:
if config.has_option(MAIL_CONFIG_SECTION, opt):
# Convert the options to the expected type
MAIL_OPTIONS[opt] = config_getters[type(
DEFAULT_MAIL_OPTIONS[opt])](MAIL_CONFIG_SECTION,
opt) # nopep8
else:
MAIL_OPTIONS[opt] = DEFAULT_MAIL_OPTIONS[opt]
else:
LOG.debug(
'Alerta mail configuration section not found in configuration file\n'
) # nopep8
# OPTIONS = defopts.copy()
if config.has_section(TELEGRAM_CONFIG_SECTION):
for opt in DEFAULT_TELEGRAM_OPTIONS:
# Convert the options to the expected type
if config.has_option(TELEGRAM_CONFIG_SECTION, opt):
TELEGRAM_OPTIONS[opt] = config_getters[type(
DEFAULT_TELEGRAM_OPTIONS[opt])](TELEGRAM_CONFIG_SECTION,
opt) # nopep8
else:
TELEGRAM_OPTIONS[opt] = DEFAULT_TELEGRAM_OPTIONS[opt]
else:
LOG.debug(
'Alerta telegram configuration section not found in configuration file\n'
) # nopep8
# OPTIONS = defopts.copy()
for opt in DEFAULT_OPTIONS:
# Convert the options to the expected type
if config.has_option("DEFAULT", opt):
OPTIONS[opt] = config_getters[type(DEFAULT_OPTIONS[opt])](
"DEFAULT", opt) # nopep8
else:
OPTIONS[opt] = DEFAULT_OPTIONS[opt]
OPTIONS['endpoint'] = os.environ.get('ALERTA_ENDPOINT') or OPTIONS[
'endpoint'] # nopep8
OPTIONS['key'] = os.environ.get('ALERTA_API_KEY') or OPTIONS['key']
if isinstance(config_file, list):
group_rules = []
for file in config_file:
group_rules.extend(parse_group_rules(file))
else:
group_rules = parse_group_rules(config_file)
if group_rules is not None:
OPTIONS['group_rules'] = group_rules
# Registering action for SIGTERM signal handling
signal.signal(signal.SIGTERM, on_sigterm)
try:
trigger = Trigger()
trigger.start()
except (SystemExit, KeyboardInterrupt):
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
from kombu.utils.debug import setup_logging
loginfo = 'DEBUG' if OPTIONS['debug'] else 'INFO'
setup_logging(loglevel=loginfo, loggers=[''])
with Connection(OPTIONS['amqp_url']) as conn:
try:
consumer = FanoutConsumer(connection=conn,
hold_time=OPTIONS['hold_time'])
consumer.run()
except (SystemExit, KeyboardInterrupt):
trigger.should_stop = True
trigger.join()
sys.exit(0)
except Exception as e:
print(str(e))
sys.exit(1)
from __future__ import with_statement
from kombu.mixins import ConsumerMixin
from kombu.utils import kwdict
from queues import task_queues
class Worker(ConsumerMixin):
def __init__(self, connection):
self.connection = connection
def get_consumers(self, Consumer, channel):
return [Consumer(queues=task_queues, callbacks=[self.process_task])]
def process_task(self, body, message):
fun = body["fun"]
args = body["args"]
kwargs = body["kwargs"]
fun(*args, **kwdict(kwargs))
message.ack()
if __name__ == "__main__":
from kombu import BrokerConnection
from kombu.utils.debug import setup_logging
setup_logging(loglevel="INFO")
with BrokerConnection("amqp://*****:*****@localhost:5672//") as conn:
Worker(conn).run()
import json
import time
from flask import current_app
from kombu.mixins import ConsumerMixin
import ujson
from listenbrainz.listen import Listen, NowPlayingListen
from listenbrainz.utils import get_fallback_connection_name
from kombu import Connection, Exchange, Queue, Consumer
from kombu.utils.debug import setup_logging
setup_logging()
class ListensDispatcher(ConsumerMixin):
def __init__(self, app, socketio):
self.app = app
self.socketio = socketio
self.connection = None
# there are two consumers, so we need two channels: one for playing now queue and another
# for normal listens queue. when using ConsumerMixin, it sets up a default channel itself.
# we create the other channel here. we also need to handle its cleanup later
self.playing_now_channel = None
self.unique_exchange = Exchange(app.config["UNIQUE_EXCHANGE"],
"fanout",
durable=False)
self.playing_now_exchange = Exchange(
parser.add_argument(
'--interval', '-i', type=int, default=30,
help='Interval to check reddit at'
)
parser.add_argument(
'--config', '-c', type=str, default='/etc/redditbot.yml',
help='Path to YAML config file'
)
parser.add_argument(
'--log', '-l', type=str, default='info',
choices=('debug', 'info', 'warning'),
help='Logging Level'
)
args = parser.parse_args()
setup_logging(loglevel=args.log.upper(), loggers=[''])
loop = asyncio.get_event_loop()
tasks = []
with open(args.config, 'r') as ymlfile:
redditbot.config = yaml.load(ymlfile)
if args.client is True:
from redditbot.client import UpdateSubreddit
for subreddit in redditbot.config['subreddits'].keys():
tasks.append(UpdateSubreddit(subreddit, args.interval))
if args.worker is True:
from redditbot import bot # noqa
from redditbot import worker
