def __init__(self, modelID):
"""
:param modelID: model ID; string
"""
self._logger = _getLogger()
self._modelID = modelID
self._swapperAPI = ModelSwapperInterface()
self._archiver = _ModelArchiver(self._modelID)
# "deleteModel" command handler sets this flag to force our processing
# loop to terminate
self._done = False
modelSwapperConfig = ModelSwapperConfig()
self._targetMaxRequestsPerCheckpoint = modelSwapperConfig.getint(
"model_runner", "target_requests_per_checkpoint")
self._profiling = (modelSwapperConfig.getboolean(
"debugging", "profiling")
or self._logger.isEnabledFor(logging.DEBUG))
if self._profiling:
self._logger.info("Profiling is turned on")
self._modelLoadSec = 0
def __init__(self, modelID):
"""
:param modelID: model ID; string
"""
self._logger = _getLogger()
self._modelID = modelID
self._swapperAPI = ModelSwapperInterface()
self._archiver = _ModelArchiver(self._modelID)
# "deleteModel" command handler sets this flag to force our processing
# loop to terminate
self._done = False
modelSwapperConfig = ModelSwapperConfig()
self._targetMaxRequestsPerCheckpoint = modelSwapperConfig.getint(
"model_runner", "target_requests_per_checkpoint")
self._profiling = (
modelSwapperConfig.getboolean("debugging", "profiling") or
self._logger.isEnabledFor(logging.DEBUG))
if self._profiling:
self._logger.info("Profiling is turned on")
self._modelLoadSec = 0
def __init__(self):
"""
Initialize the ModelSwapperInterface. This uses a lazy loading of the input
and output queues with no pre-meditation.
"""
self._logger = _getLogger()
config = ModelSwapperConfig()
self._resultsQueueName = config.get(
self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME)
# The name of a model's input message queue is the concatenation of this
# prefix and the modelID
self._modelInputQueueNamePrefix = config.get(
self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME)
self._schedulerNotificationQueueName = config.get(
self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME)
# Message bus connector
self._bus = MessageBusConnector()
# Outstanding request and/or response consumer instances
self._consumers = []
def __init__(self):
"""
Initialize the ModelSwapperInterface. This uses a lazy loading of the input
and output queues with no pre-meditation.
"""
self._logger = _getLogger()
config = ModelSwapperConfig()
self._resultsQueueName = config.get(self._CONFIG_SECTION,
self._RESULTS_Q_OPTION_NAME)
# The name of a model's input message queue is the concatenation of this
# prefix and the modelID
self._modelInputQueueNamePrefix = config.get(
self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME)
self._schedulerNotificationQueueName = config.get(
self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME)
# Message bus connector
self._bus = MessageBusConnector()
# Outstanding request and/or response consumer instances
self._consumers = []
def __init__(self, concurrency):
"""
concurrency: allowed number of model slots
"""
self._logger = _getLogger()
self._profiling = (ModelSwapperConfig().getboolean(
"debugging", "profiling")
or self._logger.isEnabledFor(logging.DEBUG))
# Allowed number of model slots
self._concurrency = concurrency
# Input-reader thread target function sets this when it starts running to
# let our event loop know that things are off to a good start
self._notificationReaderStartedEvent = threading.Event()
self._notificationMutex = threading.Lock()
# Mutex used to guaranteed that no further model input notifications will
# be added to main event queue once self._stopNotificationReader is set
# _runNotificationReaderThread will not process any more notifications
# once it detects that this flag is true
self._stopNotificationReader = False
# The event loop will exit some time after an event handler sets this flag
# to True
self._eventLoopStopPending = False
# (non-thread-safe) The tuple of all slot agents
self._slotAgents = tuple(
SlotAgent(slotID=i) for i in xrange(concurrency))
assert self._slotAgents
# Thread-safe event queue for SwapController
self._eventQ = Queue.Queue()
# Main event loop's ModelSwapperInterface instance. MUST NOT use from
# threads because ModelSwapperInterface
self._mainSwapper = ModelSwapperInterface()
# A (non-thread-safe) FIFO of models that are waiting to be scheduled for
# running; there is incoming data for them that needs to be processed
self._waitingModelsFIFO = []
# A (non-thread-safe) map of modelIDs to _RunningModelInfo instances
self._runningModelsMap = dict()
# A (non-thread-safe) list of free slot indexes into the self._slotsAgents
# tuple
self._freeSlots = list(xrange(len(self._slotAgents)))
# (non-thread-safe) Indexes of SlotAgents pending preemption
self._pendingPreemptSlotsSet = set()
self._notificationReaderThread = threading.Thread(
target=self._runNotificationReaderThread,
name="%s-input-reader-%s" % (self.__class__.__name__, id(self)))
# Allow process to exit even if thread is still running
self._notificationReaderThread.setDaemon(True)
class ModelSwapperInterface(object):
"""
This is the interface class to connect the application layer to the Model
Swapper.
"""
#_INPUT_Q_OPTION_NAME = "input_queue"
#_INPUT_Q_ENV_VAR = ModelSwapperConfig.getEnvVarOverrideName(
# configName=ModelSwapperConfig.CONFIG_NAME,
# section=_CONFIG_SECTION,
# option=_INPUT_Q_OPTION_NAME)
#""" For testing: environment variable for overriding input queue name """
_CONFIG_SECTION = "interface_bus"
_RESULTS_Q_OPTION_NAME = "results_queue"
# For testing: environment variable for overriding output queue name
_RESULTS_Q_ENV_VAR = ModelSwapperConfig()._getEnvVarOverrideName(
configName=ModelSwapperConfig.CONFIG_NAME,
section=_CONFIG_SECTION,
option=_RESULTS_Q_OPTION_NAME)
_SCHEDULER_NOTIFICATION_Q_OPTION_NAME = "scheduler_notification_queue"
_MODEL_INPUT_Q_PREFIX_OPTION_NAME = "model_input_queue_prefix"
def __init__(self):
"""
Initialize the ModelSwapperInterface. This uses a lazy loading of the input
and output queues with no pre-meditation.
"""
self._logger = _getLogger()
config = ModelSwapperConfig()
self._resultsQueueName = config.get(self._CONFIG_SECTION,
self._RESULTS_Q_OPTION_NAME)
# The name of a model's input message queue is the concatenation of this
# prefix and the modelID
self._modelInputQueueNamePrefix = config.get(
self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME)
self._schedulerNotificationQueueName = config.get(
self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME)
# Message bus connector
self._bus = MessageBusConnector()
# Outstanding request and/or response consumer instances
self._consumers = []
def __enter__(self):
""" Context Manager protocol method. Allows a ModelSwapperInterface instance
to be used in a "with" statement for automatic clean-up
Parameters:
------------------------------------------------------------------------
retval: self.
"""
return self
def __exit__(self, _excType, _excVal, _excTb):
""" Context Manager protocol method. Allows a ModelSwapperInterface instance
to be used in a "with" statement for automatic cleanup
:returns: False so as not to suppress the exception, if any
"""
self.close()
return False
def close(self):
"""
Gracefully close ModelSwapperInterface instance (e.g., tear down
connections). If this is not called, the underlying connections will
eventually timeout, but it is good practice to close explicitly.
"""
if self._consumers:
self._logger.error(
"While closing %s, discovered %s unclosed consumers; will "
"attempt to close them now", self.__class__.__name__,
len(self._consumers))
for consumer in tuple(self._consumers):
consumer.close()
assert not self._consumers
try:
self._bus.close()
finally:
self._bus = None
def _onConsumerClosed(self, consumer):
""" Called by consumer instance's close() method to remove the consumer from
our outstanding consumers list
"""
self._consumers.remove(consumer)
def _getModelInputQName(self, modelID):
return self._modelInputQueueNamePrefix + modelID
def _getModelIDFromInputQName(self, mqName):
assert mqName.startswith(self._modelInputQueueNamePrefix), (
"mq=%s doesn't start with %s") % (mqName,
self._modelInputQueueNamePrefix)
return mqName[len(self._modelInputQueueNamePrefix):]
def defineModel(self, modelID, args, commandID):
""" Initialize model's input message queue and send the "defineModel"
command. The ModelCommandResult will be delivered asynchronously, along with
the corresponding commandID and no args, to the process that is consuming
ModelSwapper results.
:param modelID: a hex string that uniquely identifies the target model.
:param args: dict with the following properties:
"modelConfig": model config dict suitable for passing to OPF
ModelFactory.create()
"inferenceArgs": Model inference arguments suitable for passing to
model.enableInference()
"inputRecordSchema": a sequence of nupic.data.fieldmeta.FieldMetaInfo
instances with field names/types/special as expected by the model and in
the same order as they will appear in input records. This is needed in
order to avoid the overhead of passing fields names with each and every
input record, while permitting the necessary dictionaries to be
constructed by ModelRunner for input to the OPF Model.
:param commandID: a numeric or string id to associate with the command and
result.
"""
# TODO: validate input args dict against schema
mqName = self._getModelInputQName(modelID)
self._bus.createMessageQueue(mqName, durable=True)
self.submitRequests(modelID,
(ModelCommand(commandID, "defineModel", args), ))
def cloneModel(self, modelID, newModelID, commandID):
""" Initiate cloning of an existing model. Initialize the new
model's input message queue and send the "cloneModel" command to the source
model. The ModelCommandResult will be delivered asynchronously, along with
the corresponding commandID and no args, to the process that is consuming
ModelSwapper results.
:param modelID: a hex string that uniquely identifies the existing model.
:param newModelID: a hex string that uniquely identifies the new model.
:param commandID: a numeric or string id to associate with the command and
result.
:raises: ModelNotFound if the source model's input endpoint doesn't exist
"""
# Create the model input message queue for the new model
self._bus.createMessageQueue(self._getModelInputQName(newModelID),
durable=True)
self.submitRequests(modelID, (ModelCommand(
commandID, "cloneModel", args={"modelID": newModelID}), ))
def deleteModel(self, modelID, commandID):
""" Submit a request to delete a model. The ModelCommandResult will be
delivered asynchronously, along with the corresponding commandID and no
args, to the process that is consuming ModelSwapper results.
This method is idempotent.
:param modelID: a hex string that uniquely identifies the target model.
:param commandID: a numeric or string id to associate with the command and
result.
"""
# First, purge unread input messages for this model, if any, to avoid
# unnecessary processing before the model is deleted
mq = self._getModelInputQName(modelID)
self._logger.info(
"deleteModel: purging mq=%s before submitting "
"deleteModel command for model=%s", mq, modelID)
try:
self._bus.purge(mq)
except message_bus_connector.MessageQueueNotFound:
# deleteModel is an idempotent operation: assume this exception is
# due to repeated attempt
pass
else:
try:
self.submitRequests(modelID,
(ModelCommand(commandID, "deleteModel"), ))
except ModelNotFound:
# deleteModel is an idempotent operation: assume this exception is
# due to repeated attempt
pass
def cleanUpAfterModelDeletion(self, modelID):
""" For use by Engine's ModelRunner after it deletes a model: clean up
resources that ModelSwapperInterface created to support the model, such
as deleting the model's input message queue
"""
self._bus.deleteMessageQueue(self._getModelInputQName(modelID))
def modelInputPending(self, modelID):
""" Check if input requests are pending for a model
:param modelID: a string that uniquely identifies the target model.
:returns: True if the model's input queue exists and is non-empty;
False if the model's input queue is non-empty or doesn't exist
"""
try:
return not self._bus.isEmpty(self._getModelInputQName(modelID))
except message_bus_connector.MessageQueueNotFound:
return False
def getModelsWithInputPending(self):
""" Get model IDs of all models with pending input (non-empty input queues)
:returns: (possibly empty) sequence of model IDs whose input streams are
non-empty
"""
# NOTE: queues may be deleted as we're running through the list, so we need
# to play it safe
def safeIsInputPending(mq):
try:
return not self._bus.isEmpty(mq)
except message_bus_connector.MessageQueueNotFound:
return False
prefix = self._modelInputQueueNamePrefix
return tuple(
self._getModelIDFromInputQName(mq)
for mq in self._bus.getAllMessageQueues()
if mq.startswith(prefix) and safeIsInputPending(mq))
def submitRequests(self, modelID, requests):
"""
Submit a batch of requests for processing by a model with the given modelID.
NOTE: it's an error to submit requests for a model after calling
deleteModel()
Keyword arguments:
:param modelID: a string that uniquely identifies the target model.
:param requests: a sequence of ModelCommand and/or ModelInputRow instances.
NOTE: To create or delete a model, call the createModel or deleteModel
method instead of submitting the "defineModel" or "deleteModel" commands.
Together, the sequence of requests constitutes a request "batch".
:returns: UUID of the submitted batch (intended for test code only)
:raises: ModelNotFound if model's input endpoint doesn't exist
Requests for a specific model will be processed in the submitted order.
The results will be delivered asynchronously, along with the corresponding
requestIDs, to the process that is consuming ModelSwapper results.
NOTE: This assumes retry logic will be handled by the underlying MQ
implementation.
"""
batchID = uuid.uuid1().hex
msg = RequestMessagePackager.marshal(
batchID=batchID, batchState=BatchPackager.marshal(batch=requests))
mqName = self._getModelInputQName(modelID)
try:
self._bus.publish(mqName, msg, persistent=True)
except message_bus_connector.MessageQueueNotFound as e:
self._logger.warn(
"App layer attempted to submit numRequests=%s to model=%s, but its "
"input queue doesn't exist. Likely a race condition with model "
"deletion path.", len(requests), modelID)
raise ModelNotFound(repr(e))
except:
self._logger.exception(
"Failed to publish request batch=%s for model=%s via mq=%s; "
"msgLen=%s; msgPrefix=%r. NOTE: it's an error to submit requests to a "
"model after deleting it.", batchID, modelID, mqName, len(msg),
msg[:32])
raise
# Send a notification to Model Scheduler so it will schedule the model
# for processing input
try:
self._bus.publish(self._schedulerNotificationQueueName,
json.dumps(modelID),
persistent=False)
except message_bus_connector.MessageQueueNotFound:
# If it's not fully up yet, its notification queue might not have been
# created, which is ok
self._logger.warn(
"Couldn't send model data notification to Model Scheduler: mq=%s not "
"found. Model Scheduler service not started or initialized the mq yet?",
self._schedulerNotificationQueueName)
return batchID
def consumeRequests(self, modelID, blocking=True):
""" Create an instance of the _MessageConsumer iterable for reading model
requests, a batch at a time. The iterable yields _ConsumedRequestBatch
instances.
NOTE: This API is intended for Engine Model Runners.
:param modelID: a string that uniquely identifies the target model.
:param blocking: if True, the iterable will block until another batch becomes
available; if False, the iterable will terminate iteration when no more
batches are available in the queue. [defaults to True]
:returns: an instance of model_swapper_interface._MessageConsumer iterable;
IMPORTANT: the caller is responsible for closing it before closing this
ModelSwapperInterface instance (hint: use the returned _MessageConsumer
instance as Context Manager)
:raises: ModelNotFound if model's input endpoint doesn't exist
TODO: need tests for consumeRequests with ModelNotFound
Example:
with ModelSwapperInterface() as swapper:
with swapper.consumeRequests(modelID) as consumer:
for batch in consumer:
processRequests(batchID=batch.batchID, requests=batch.objects)
batch.ack()
"""
mq = self._getModelInputQName(modelID)
def onQueueNotFound():
msg = (
"Attempt to consume requests from model=%s is impossible because "
"its input queue doesn't exist. Likely a race condition with "
"model deletion path.") % (modelID, )
self._logger.warn(msg)
raise ModelNotFound(msg)
consumer = _MessageConsumer(mqName=mq,
blocking=blocking,
decode=_ConsumedRequestBatch.decodeMessage,
swapper=self,
bus=self._bus,
onQueueNotFound=onQueueNotFound)
self._consumers.append(consumer)
return consumer
def _initResultsMessageQueue(self):
self._bus.createMessageQueue(self._resultsQueueName, durable=True)
def submitResults(self, modelID, results):
"""
Submit a batch of results (used by ModelSwapper layer)
Keyword arguments:
:param modelID: a string that uniquely identifies the target model.
:param results: a sequence of ModelCommandResult and/or ModelInferenceResult
instances
NOTE: This assumes retry logic will be handled by the underlying MQ
implementation.
"""
msg = ResultMessagePackager.marshal(
modelID=modelID, batchState=BatchPackager.marshal(batch=results))
try:
try:
self._bus.publish(self._resultsQueueName, msg, persistent=True)
except message_bus_connector.MessageQueueNotFound:
self._logger.info(
"submitResults: results mq=%s didn't exist; "
"declaring now and re-publishing message",
self._resultsQueueName)
self._initResultsMessageQueue()
self._bus.publish(self._resultsQueueName, msg, persistent=True)
except:
self._logger.exception(
"submitResults: Failed to publish results from model=%s via mq=%s; "
"msgLen=%s; msgPrefix=%r", modelID, self._resultsQueueName,
len(msg), msg[:32])
raise
def consumeResults(self):
""" Create an instance of the _MessageConsumer iterable for reading model
results, a batch at a time. The iterable yields _ConsumedResultBatch
instances.
:returns: an instance of model_swapper_interface._MessageConsumer iterable;
IMPORTANT: the caller is responsible for closing it before closing this
ModelSwapperInterface instance (hint: use the returned _MessageConsumer
instance as Context Manager)
Example:
with ModelSwapperInterface() as swapper:
with swapper.consumeResults() as consumer:
for batch in consumer:
processResults(modelID=batch.modelID, results=batch.objects)
batch.ack()
"""
consumer = _MessageConsumer(
mqName=self._resultsQueueName,
blocking=True,
decode=_ConsumedResultBatch.decodeMessage,
swapper=self,
bus=self._bus,
onQueueNotFound=self._initResultsMessageQueue)
self._consumers.append(consumer)
return consumer
def initSchedulerNotification(self):
""" Initialize Model Scheduler's notification message queue; for use by
Model Scheduler.
"""
self._bus.createMessageQueue(self._schedulerNotificationQueueName,
durable=False)
def consumeModelSchedulerNotifications(self):
""" Create an instance of the _MessageConsumer iterable for reading model
scheduler notifications. The iterable yields _ConsumedNotification
instances.
:returns: an instance of model_swapper_interface._MessageConsumer iterable;
IMPORTANT: the caller is responsible for closing it before closing this
ModelSwapperInterface instance (hint: use the returned _MessageConsumer
instance as Context Manager)
Example:
with ModelSwapperInterface() as swapper:
with swapper.consumeModelSchedulerNotifications() as consumer:
for notification in consumer:
processNotification(notification.value)
notification.ack()
"""
consumer = _MessageConsumer(
mqName=self._schedulerNotificationQueueName,
blocking=True,
decode=_ConsumedNotification.decodeMessage,
swapper=self,
bus=self._bus)
self._consumers.append(consumer)
return consumer
def _parseArgs():
"""
:returns: dict of arg names and values:
rmqHost: Host of RabbitMQ management interface
rmqHost: Port number of RabbitMQ management interface
rmqUser: RabbitMQ username
rmqPassword: RabbitMQ password
rmqQueues: sequence of vhost-qualified RabbitMQ queue names to monitor
e.g., ["%2f/taurus.metric.custom.data",
"%2f/taurus.mswapper.results",
"%2f/taurus.mswapper.scheduler.notification"]
metricDestHost: Host of metric destination address; None for dry-run
metricDestPort: Port number of metric destination address
metricPrefix: prefix for emitted metric names
"""
usage = (
"%prog [options]\n\n"
"Collects statistics from a RabbitMQ server and emits them "
"as metrics to the destination Grok server.\n"
"\n"
"The following metrics are collected and emitted by default, where\n"
"<prefix> is the value of the --metric-prefix command-line option.\n"
"\t<prefix>-allq-ready.avg - average number of READY messages in all\n"
"\t\tqueues.\n"
"\n"
"\t<prefix>-q-taurus.metric.custom.data-ready.avg - average number of\n"
"\t\tREADY messages in htmengine's Metric Storer input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.results-ready.avg - average number of READY\n"
"\t\tmessages in htmengine's Anomaly Service input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.scheduler.notification-ready.avg - average\n"
"\t\tnumber of READY messages in htmengine's Model Scheduler notification\n"
"\t\tinput queue"
)
parser = OptionParser(usage=usage)
# Get params to use as option defaults
rmqParams = RabbitmqManagementConnectionParams()
parser.add_option(
"--rmq-addr",
action="store",
type="string",
dest="rmqAddr",
default="%s:%d" % (rmqParams.host, rmqParams.port),
help=("Address and port host:port of RabbitMQ Management interface "
"[default: %default]"))
parser.add_option(
"--rmq-user",
action="store",
type="string",
dest="rmqUser",
default=rmqParams.username,
help="Username for RabbitMQ authentication [default: %default]")
parser.add_option(
"--rmq-pass",
action="store",
type="string",
dest="rmqPassword",
default=rmqParams.password,
help="Password for RabbitMQ authentication [default: %default]")
rmqVhost = (rmqParams.vhost if rmqParams.vhost != "/"
else "%" + rmqParams.vhost.encode("hex"))
appConfig = Config("application.conf", os.environ.get("APPLICATION_CONFIG_PATH"))
swapperConfig = ModelSwapperConfig()
defaultQueues = [
swapperConfig.get("interface_bus", "results_queue"),
swapperConfig.get("interface_bus", "scheduler_notification_queue"),
appConfig.get("metric_listener", "queue_name")
]
defaultQueues = ["%s/%s" % (rmqVhost, q) for q in defaultQueues]
parser.add_option(
"--rmq-queues",
action="store",
type="string",
dest="rmqQueues",
default=",".join(defaultQueues),
help=("RabbitMQ message queues to monitor; comma-separated, "
"vhost-qualified; [default: %default]"))
parser.add_option(
"--dryrun",
action="store_true",
default=False,
dest="dryRun",
help=("Use this flag to do a dry run: retrieve data and log it; mutually "
"exclusive with --metric-addr"))
parser.add_option(
"--metric-addr",
action="store",
type="string",
dest="metricDestAddr",
help=("Destination address for metrics as host:port; typically address of "
"Grok's custom metrics listener; Grok's default metric listener port "
"is 2003"))
parser.add_option(
"--metric-prefix",
action="store",
type="string",
dest="metricPrefix",
help="Prefix for metric names")
options, remainingArgs = parser.parse_args()
if remainingArgs:
msg = "Unexpected remaining args: %r" % (remainingArgs,)
g_log.error(msg)
parser.error(msg)
if not options.rmqAddr:
msg = "Missing address of RabbitMQ server"
g_log.error(msg)
parser.error(msg)
rmqHost, _, rmqPort = options.rmqAddr.rpartition(":")
if not rmqHost:
msg = "Missing Hostname or IP address of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
if not rmqPort:
msg = "Missing port number of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
try:
rmqPort = int(rmqPort)
except ValueError:
msg = ("RabbitMQ Management Interface port must be an integer, but got %r"
% (metricDestPort,))
g_log.exception(msg)
parser.error(msg)
if not options.rmqUser:
msg = "Missing RabbitMQ user name."
g_log.error(msg)
parser.error(msg)
if not options.rmqPassword:
msg = "Missing RabbitMQ password."
g_log.error(msg)
parser.error(msg)
if not options.rmqQueues:
msg = "Missing vhost-qualified message queue names"
g_log.error(msg)
parser.error(msg)
rmqQueues = options.rmqQueues.split(",")
if options.dryRun:
if options.metricDestAddr:
msg = "--dryrun is mutually exclusive with --metric-addr"
g_log.error(msg)
parser.error(msg)
metricDestHost = metricDestPort = None
else:
if not options.metricDestAddr:
msg = "Missing address of metric destination server"
g_log.error(msg)
parser.error(msg)
metricDestHost, _, metricDestPort = options.metricDestAddr.rpartition(":")
if not metricDestHost:
msg = "Missing Hostname or IP address of metric destination server."
g_log.error(msg)
parser.error(msg)
if not metricDestPort:
msg = "Missing port number of metric destination server."
g_log.error(msg)
parser.error(msg)
try:
metricDestPort = int(metricDestPort)
except ValueError:
msg = "Metric destination port must be an integer, but got %r" % (
metricDestPort,)
g_log.exception(msg)
parser.error(msg)
options.metricPrefix = (options.metricPrefix.strip()
if options.metricPrefix is not None else None)
if not options.metricPrefix:
msg = "Missing or empty metric name prefix"
g_log.error(msg)
parser.error(msg)
return dict(
rmqHost=rmqHost,
rmqPort=rmqPort,
rmqUser=options.rmqUser,
rmqPassword=options.rmqPassword,
rmqQueues=rmqQueues,
metricDestHost=metricDestHost,
metricDestPort=metricDestPort,
metricPrefix=options.metricPrefix
)
def _parseArgs():
"""
:returns: dict of arg names and values:
rmqHost: Host of RabbitMQ management interface
rmqHost: Port number of RabbitMQ management interface
rmqUser: RabbitMQ username
rmqPassword: RabbitMQ password
rmqQueues: sequence of vhost-qualified RabbitMQ queue names to monitor
e.g., ["%2f/taurus.metric.custom.data",
"%2f/taurus.mswapper.results",
"%2f/taurus.mswapper.scheduler.notification"]
metricDestHost: Host of metric destination address; None for dry-run
metricDestPort: Port number of metric destination address
metricPrefix: prefix for emitted metric names
"""
usage = (
"%prog [options]\n\n"
"Collects statistics from a RabbitMQ server and emits them "
"as metrics to the destination htmengine app server.\n"
"\n"
"The following metrics are collected and emitted by default, where\n"
"<prefix> is the value of the --metric-prefix command-line option.\n"
"\t<prefix>-allq-ready.avg - average number of READY messages in all\n"
"\t\tqueues.\n"
"\n"
"\t<prefix>-q-taurus.metric.custom.data-ready.avg - average number of\n"
"\t\tREADY messages in htmengine's Metric Storer input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.results-ready.avg - average number of READY\n"
"\t\tmessages in htmengine's Anomaly Service input queue.\n"
"\n"
"\t<prefix>-q-taurus.mswapper.scheduler.notification-ready.avg - average\n"
"\t\tnumber of READY messages in htmengine's Model Scheduler notification\n"
"\t\tinput queue"
)
parser = OptionParser(usage=usage)
# Get params to use as option defaults
rmqParams = amqp.connection.RabbitmqManagementConnectionParams()
parser.add_option(
"--rmq-addr",
action="store",
type="string",
dest="rmqAddr",
default="%s:%d" % (rmqParams.host, rmqParams.port),
help=("Address and port host:port of RabbitMQ Management interface "
"[default: %default]"))
parser.add_option(
"--rmq-user",
action="store",
type="string",
dest="rmqUser",
default=rmqParams.username,
help="Username for RabbitMQ authentication [default: %default]")
parser.add_option(
"--rmq-pass",
action="store",
type="string",
dest="rmqPassword",
default=rmqParams.password,
help="Password for RabbitMQ authentication [default: %default]")
rmqVhost = (rmqParams.vhost if rmqParams.vhost != "/"
else "%" + rmqParams.vhost.encode("hex"))
appConfig = Config("application.conf", os.environ.get("APPLICATION_CONFIG_PATH"))
swapperConfig = ModelSwapperConfig()
defaultQueues = [
swapperConfig.get("interface_bus", "results_queue"),
swapperConfig.get("interface_bus", "scheduler_notification_queue"),
appConfig.get("metric_listener", "queue_name")
]
defaultQueues = ["%s/%s" % (rmqVhost, q) for q in defaultQueues]
parser.add_option(
"--rmq-queues",
action="store",
type="string",
dest="rmqQueues",
default=",".join(defaultQueues),
help=("RabbitMQ message queues to monitor; comma-separated, "
"vhost-qualified; [default: %default]"))
parser.add_option(
"--dryrun",
action="store_true",
default=False,
dest="dryRun",
help=("Use this flag to do a dry run: retrieve data and log it; mutually "
"exclusive with --metric-addr"))
parser.add_option(
"--metric-addr",
action="store",
type="string",
dest="metricDestAddr",
help=("Destination address for metrics as host:port; typically address of "
"htmengine custom metrics listener; htmengine default metric "
"listener port is 2003"))
parser.add_option(
"--metric-prefix",
action="store",
type="string",
dest="metricPrefix",
help="Prefix for metric names")
options, remainingArgs = parser.parse_args()
if remainingArgs:
msg = "Unexpected remaining args: %r" % (remainingArgs,)
g_log.error(msg)
parser.error(msg)
if not options.rmqAddr:
msg = "Missing address of RabbitMQ server"
g_log.error(msg)
parser.error(msg)
rmqHost, _, rmqPort = options.rmqAddr.rpartition(":")
if not rmqHost:
msg = "Missing Hostname or IP address of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
if not rmqPort:
msg = "Missing port number of RabbitMQ management interface."
g_log.error(msg)
parser.error(msg)
try:
rmqPort = int(rmqPort)
except ValueError:
msg = ("RabbitMQ Management Interface port must be an integer, but got %r"
% (metricDestPort,))
g_log.exception(msg)
parser.error(msg)
if not options.rmqUser:
msg = "Missing RabbitMQ user name."
g_log.error(msg)
parser.error(msg)
if not options.rmqPassword:
msg = "Missing RabbitMQ password."
g_log.error(msg)
parser.error(msg)
if not options.rmqQueues:
msg = "Missing vhost-qualified message queue names"
g_log.error(msg)
parser.error(msg)
rmqQueues = options.rmqQueues.split(",")
if options.dryRun:
if options.metricDestAddr:
msg = "--dryrun is mutually exclusive with --metric-addr"
g_log.error(msg)
parser.error(msg)
metricDestHost = metricDestPort = None
else:
if not options.metricDestAddr:
msg = "Missing address of metric destination server"
g_log.error(msg)
parser.error(msg)
metricDestHost, _, metricDestPort = options.metricDestAddr.rpartition(":")
if not metricDestHost:
msg = "Missing Hostname or IP address of metric destination server."
g_log.error(msg)
parser.error(msg)
if not metricDestPort:
msg = "Missing port number of metric destination server."
g_log.error(msg)
parser.error(msg)
try:
metricDestPort = int(metricDestPort)
except ValueError:
msg = "Metric destination port must be an integer, but got %r" % (
metricDestPort,)
g_log.exception(msg)
parser.error(msg)
options.metricPrefix = (options.metricPrefix.strip()
if options.metricPrefix is not None else None)
if not options.metricPrefix:
msg = "Missing or empty metric name prefix"
g_log.error(msg)
parser.error(msg)
return dict(
rmqHost=rmqHost,
rmqPort=rmqPort,
rmqUser=options.rmqUser,
rmqPassword=options.rmqPassword,
rmqQueues=rmqQueues,
metricDestHost=metricDestHost,
metricDestPort=metricDestPort,
metricPrefix=options.metricPrefix
)