def __init__(self):
"""
Initialize the ModelSwapperInterface. This uses a lazy loading of the input
and output queues with no pre-meditation.
"""
self._logger = _getLogger()
config = ModelSwapperConfig()
self._resultsQueueName = config.get(
self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME)
# The name of a model's input message queue is the concatenation of this
# prefix and the modelID
self._modelInputQueueNamePrefix = config.get(
self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME)
self._schedulerNotificationQueueName = config.get(
self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME)
# Message bus connector
self._bus = MessageBusConnector()
# Outstanding request and/or response consumer instances
self._consumers = []
def __init__(self):
"""
Initialize the ModelSwapperInterface. This uses a lazy loading of the input
and output queues with no pre-meditation.
"""
self._logger = _getLogger()
config = ModelSwapperConfig()
self._resultsQueueName = config.get(self._CONFIG_SECTION,
self._RESULTS_Q_OPTION_NAME)
# The name of a model's input message queue is the concatenation of this
# prefix and the modelID
self._modelInputQueueNamePrefix = config.get(
self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME)
self._schedulerNotificationQueueName = config.get(
self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME)
# Message bus connector
self._bus = MessageBusConnector()
# Outstanding request and/or response consumer instances
self._consumers = []
def _parseArgs():
"""
:returns: dict of arg names and values:
rmqHost: Host of RabbitMQ management interface
rmqHost: Port number of RabbitMQ management interface
rmqUser: RabbitMQ username
rmqPassword: RabbitMQ password
rmqQueues: sequence of vhost-qualified RabbitMQ queue names to monitor
e.g., ["%2f/taurus.metric.custom.data",
"%2f/taurus.mswapper.results",
"%2f/taurus.mswapper.scheduler.notification"]
metricDestHost: Host of metric destination address; None for dry-run
metricDestPort: Port number of metric destination address
metricPrefix: prefix for emitted metric names
"""
usage = (
"%prog [options]\n\n"
"Collects statistics from a RabbitMQ server and emits them "
"as metrics to the destination Grok server.\n"
"\n"
"The following metrics are collected and emitted by default, where\n"
"<prefix> is the value of the --metric-prefix command-line option.\n"
"\t<prefix>-allq-ready.avg - average number of READY messages in all\n"
"\t\tqueues.\n"
"\n"
"\t<prefix>-q-taurus.metric.custom.data-ready.avg - average number of\n"
"\t\tREADY messages in htmengine's Metric Storer input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.results-ready.avg - average number of READY\n"
"\t\tmessages in htmengine's Anomaly Service input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.scheduler.notification-ready.avg - average\n"
"\t\tnumber of READY messages in htmengine's Model Scheduler notification\n"
"\t\tinput queue"
)
parser = OptionParser(usage=usage)
# Get params to use as option defaults
rmqParams = RabbitmqManagementConnectionParams()
parser.add_option(
"--rmq-addr",
action="store",
type="string",
dest="rmqAddr",
default="%s:%d" % (rmqParams.host, rmqParams.port),
help=("Address and port host:port of RabbitMQ Management interface "
"[default: %default]"))
parser.add_option(
"--rmq-user",
action="store",
type="string",
dest="rmqUser",
default=rmqParams.username,
help="Username for RabbitMQ authentication [default: %default]")
parser.add_option(
"--rmq-pass",
action="store",
type="string",
dest="rmqPassword",
default=rmqParams.password,
help="Password for RabbitMQ authentication [default: %default]")
rmqVhost = (rmqParams.vhost if rmqParams.vhost != "/"
else "%" + rmqParams.vhost.encode("hex"))
appConfig = Config("application.conf", os.environ.get("APPLICATION_CONFIG_PATH"))
swapperConfig = ModelSwapperConfig()
defaultQueues = [
swapperConfig.get("interface_bus", "results_queue"),
swapperConfig.get("interface_bus", "scheduler_notification_queue"),
appConfig.get("metric_listener", "queue_name")
]
defaultQueues = ["%s/%s" % (rmqVhost, q) for q in defaultQueues]
parser.add_option(
"--rmq-queues",
action="store",
type="string",
dest="rmqQueues",
default=",".join(defaultQueues),
help=("RabbitMQ message queues to monitor; comma-separated, "
"vhost-qualified; [default: %default]"))
parser.add_option(
"--dryrun",
action="store_true",
default=False,
dest="dryRun",
help=("Use this flag to do a dry run: retrieve data and log it; mutually "
"exclusive with --metric-addr"))
parser.add_option(
"--metric-addr",
action="store",
type="string",
dest="metricDestAddr",
help=("Destination address for metrics as host:port; typically address of "
"Grok's custom metrics listener; Grok's default metric listener port "
"is 2003"))
parser.add_option(
"--metric-prefix",
action="store",
type="string",
dest="metricPrefix",
help="Prefix for metric names")
options, remainingArgs = parser.parse_args()
if remainingArgs:
msg = "Unexpected remaining args: %r" % (remainingArgs,)
g_log.error(msg)
parser.error(msg)
if not options.rmqAddr:
msg = "Missing address of RabbitMQ server"
g_log.error(msg)
parser.error(msg)
rmqHost, _, rmqPort = options.rmqAddr.rpartition(":")
if not rmqHost:
msg = "Missing Hostname or IP address of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
if not rmqPort:
msg = "Missing port number of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
try:
rmqPort = int(rmqPort)
except ValueError:
msg = ("RabbitMQ Management Interface port must be an integer, but got %r"
% (metricDestPort,))
g_log.exception(msg)
parser.error(msg)
if not options.rmqUser:
msg = "Missing RabbitMQ user name."
g_log.error(msg)
parser.error(msg)
if not options.rmqPassword:
msg = "Missing RabbitMQ password."
g_log.error(msg)
parser.error(msg)
if not options.rmqQueues:
msg = "Missing vhost-qualified message queue names"
g_log.error(msg)
parser.error(msg)
rmqQueues = options.rmqQueues.split(",")
if options.dryRun:
if options.metricDestAddr:
msg = "--dryrun is mutually exclusive with --metric-addr"
g_log.error(msg)
parser.error(msg)
metricDestHost = metricDestPort = None
else:
if not options.metricDestAddr:
msg = "Missing address of metric destination server"
g_log.error(msg)
parser.error(msg)
metricDestHost, _, metricDestPort = options.metricDestAddr.rpartition(":")
if not metricDestHost:
msg = "Missing Hostname or IP address of metric destination server."
g_log.error(msg)
parser.error(msg)
if not metricDestPort:
msg = "Missing port number of metric destination server."
g_log.error(msg)
parser.error(msg)
try:
metricDestPort = int(metricDestPort)
except ValueError:
msg = "Metric destination port must be an integer, but got %r" % (
metricDestPort,)
g_log.exception(msg)
parser.error(msg)
options.metricPrefix = (options.metricPrefix.strip()
if options.metricPrefix is not None else None)
if not options.metricPrefix:
msg = "Missing or empty metric name prefix"
g_log.error(msg)
parser.error(msg)
return dict(
rmqHost=rmqHost,
rmqPort=rmqPort,
rmqUser=options.rmqUser,
rmqPassword=options.rmqPassword,
rmqQueues=rmqQueues,
metricDestHost=metricDestHost,
metricDestPort=metricDestPort,
metricPrefix=options.metricPrefix
)
def _parseArgs():
"""
:returns: dict of arg names and values:
rmqHost: Host of RabbitMQ management interface
rmqHost: Port number of RabbitMQ management interface
rmqUser: RabbitMQ username
rmqPassword: RabbitMQ password
rmqQueues: sequence of vhost-qualified RabbitMQ queue names to monitor
e.g., ["%2f/taurus.metric.custom.data",
"%2f/taurus.mswapper.results",
"%2f/taurus.mswapper.scheduler.notification"]
metricDestHost: Host of metric destination address; None for dry-run
metricDestPort: Port number of metric destination address
metricPrefix: prefix for emitted metric names
"""
usage = (
"%prog [options]\n\n"
"Collects statistics from a RabbitMQ server and emits them "
"as metrics to the destination htmengine app server.\n"
"\n"
"The following metrics are collected and emitted by default, where\n"
"<prefix> is the value of the --metric-prefix command-line option.\n"
"\t<prefix>-allq-ready.avg - average number of READY messages in all\n"
"\t\tqueues.\n"
"\n"
"\t<prefix>-q-taurus.metric.custom.data-ready.avg - average number of\n"
"\t\tREADY messages in htmengine's Metric Storer input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.results-ready.avg - average number of READY\n"
"\t\tmessages in htmengine's Anomaly Service input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.scheduler.notification-ready.avg - average\n"
"\t\tnumber of READY messages in htmengine's Model Scheduler notification\n"
"\t\tinput queue"
)
parser = OptionParser(usage=usage)
# Get params to use as option defaults
rmqParams = amqp.connection.RabbitmqManagementConnectionParams()
parser.add_option(
"--rmq-addr",
action="store",
type="string",
dest="rmqAddr",
default="%s:%d" % (rmqParams.host, rmqParams.port),
help=("Address and port host:port of RabbitMQ Management interface "
"[default: %default]"))
parser.add_option(
"--rmq-user",
action="store",
type="string",
dest="rmqUser",
default=rmqParams.username,
help="Username for RabbitMQ authentication [default: %default]")
parser.add_option(
"--rmq-pass",
action="store",
type="string",
dest="rmqPassword",
default=rmqParams.password,
help="Password for RabbitMQ authentication [default: %default]")
rmqVhost = (rmqParams.vhost if rmqParams.vhost != "/"
else "%" + rmqParams.vhost.encode("hex"))
appConfig = Config("application.conf", os.environ.get("APPLICATION_CONFIG_PATH"))
swapperConfig = ModelSwapperConfig()
defaultQueues = [
swapperConfig.get("interface_bus", "results_queue"),
swapperConfig.get("interface_bus", "scheduler_notification_queue"),
appConfig.get("metric_listener", "queue_name")
]
defaultQueues = ["%s/%s" % (rmqVhost, q) for q in defaultQueues]
parser.add_option(
"--rmq-queues",
action="store",
type="string",
dest="rmqQueues",
default=",".join(defaultQueues),
help=("RabbitMQ message queues to monitor; comma-separated, "
"vhost-qualified; [default: %default]"))
parser.add_option(
"--dryrun",
action="store_true",
default=False,
dest="dryRun",
help=("Use this flag to do a dry run: retrieve data and log it; mutually "
"exclusive with --metric-addr"))
parser.add_option(
"--metric-addr",
action="store",
type="string",
dest="metricDestAddr",
help=("Destination address for metrics as host:port; typically address of "
"htmengine custom metrics listener; htmengine default metric "
"listener port is 2003"))
parser.add_option(
"--metric-prefix",
action="store",
type="string",
dest="metricPrefix",
help="Prefix for metric names")
options, remainingArgs = parser.parse_args()
if remainingArgs:
msg = "Unexpected remaining args: %r" % (remainingArgs,)
g_log.error(msg)
parser.error(msg)
if not options.rmqAddr:
msg = "Missing address of RabbitMQ server"
g_log.error(msg)
parser.error(msg)
rmqHost, _, rmqPort = options.rmqAddr.rpartition(":")
if not rmqHost:
msg = "Missing Hostname or IP address of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
if not rmqPort:
msg = "Missing port number of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
try:
rmqPort = int(rmqPort)
except ValueError:
msg = ("RabbitMQ Management Interface port must be an integer, but got %r"
% (metricDestPort,))
g_log.exception(msg)
parser.error(msg)
if not options.rmqUser:
msg = "Missing RabbitMQ user name."
g_log.error(msg)
parser.error(msg)
if not options.rmqPassword:
msg = "Missing RabbitMQ password."
g_log.error(msg)
parser.error(msg)
if not options.rmqQueues:
msg = "Missing vhost-qualified message queue names"
g_log.error(msg)
parser.error(msg)
rmqQueues = options.rmqQueues.split(",")
if options.dryRun:
if options.metricDestAddr:
msg = "--dryrun is mutually exclusive with --metric-addr"
g_log.error(msg)
parser.error(msg)
metricDestHost = metricDestPort = None
else:
if not options.metricDestAddr:
msg = "Missing address of metric destination server"
g_log.error(msg)
parser.error(msg)
metricDestHost, _, metricDestPort = options.metricDestAddr.rpartition(":")
if not metricDestHost:
msg = "Missing Hostname or IP address of metric destination server."
g_log.error(msg)
parser.error(msg)
if not metricDestPort:
msg = "Missing port number of metric destination server."
g_log.error(msg)
parser.error(msg)
try:
metricDestPort = int(metricDestPort)
except ValueError:
msg = "Metric destination port must be an integer, but got %r" % (
metricDestPort,)
g_log.exception(msg)
parser.error(msg)
options.metricPrefix = (options.metricPrefix.strip()
if options.metricPrefix is not None else None)
if not options.metricPrefix:
msg = "Missing or empty metric name prefix"
g_log.error(msg)
parser.error(msg)
return dict(
rmqHost=rmqHost,
rmqPort=rmqPort,
rmqUser=options.rmqUser,
rmqPassword=options.rmqPassword,
rmqQueues=rmqQueues,
metricDestHost=metricDestHost,
metricDestPort=metricDestPort,
metricPrefix=options.metricPrefix
)
