def bulk_walk(self, oid, non_repeaters=0, max_repetitions=10):
"""
Takes a single numeric oid and fetches oids under it's subtree using GETBULKs
Args:
oid (str): A numeric oid
non_repeaters (int): A non-negative integer
max_repetitions (int): A non-negative integer
Returns:
list(PanoptesSNMPVariable): A list of PanoptesSNMPVariables within the subtree of the provided oid
"""
assert PanoptesValidators.valid_numeric_snmp_oid(oid), u'oid must be numeric string with a leading period'
assert PanoptesValidators.valid_positive_integer(non_repeaters), u'non_repeaters must a positive integer'
assert PanoptesValidators.valid_positive_integer(max_repetitions), u'max_repetitions must a positive integer'
assert self._easy_snmp_session is not None, u'PanoptesSNMPSession not initialized'
base_oid = oid
varbinds = list()
running = 1
while running:
results = self.get_bulk(oid=oid, non_repeaters=non_repeaters,
max_repetitions=max_repetitions, base_oid=base_oid)
varbinds.extend(results)
if len(results) < max_repetitions:
break
oid = base_oid + u'.' + results[-1].index
return varbinds
def get_bulk(self, oid, non_repeaters=0, max_repetitions=10, base_oid=None):
"""
Takes a single numeric oid and returns a list of PanoptesSNMPVariables which start with the oid
Args:
oid (str): A numeric oid
non_repeaters (int): A non-negative integer
max_repetitions (int): A non-negative integer
base_oid (str): An optional numeric oid which is used to set the oid in the returned PanoptesSNMPVariables
Returns:
list(): A list of PanoptesSNMPVariables starting with the provided oid
"""
assert PanoptesValidators.valid_numeric_snmp_oid(oid), u'oid must be numeric string with a leading period'
assert PanoptesValidators.valid_positive_integer(non_repeaters), u'non_repeaters must a positive integer'
assert PanoptesValidators.valid_positive_integer(max_repetitions), u'max_repetitions must a positive integer'
assert (base_oid is None) or PanoptesValidators.valid_numeric_snmp_oid(
base_oid), u'oid must be numeric string with a leading period'
assert self._easy_snmp_session is not None, u'PanoptesSNMPSession not initialized'
try:
varbinds = self._easy_snmp_session.get_bulk(oids=oid, non_repeaters=non_repeaters,
max_repetitions=max_repetitions)
result = list()
for varbind in varbinds:
if varbind.oid.startswith(base_oid):
result.append(PanoptesSNMPVariable(queried_oid=base_oid if base_oid else oid, oid=varbind.oid,
index=varbind.oid_index, value=varbind.value,
snmp_type=varbind.snmp_type))
return result
except Exception as e:
raise self._wrap_panoptes_snmp_exception(e)
def _parse_x509_configuration(self):
"""
Parses the config spec to provide some details on the x509 communication. This is effectively requests over
x509 based on the value of _x509_secure_connection.
0 - x509 communications
1 - x509 where available (optional)
2 - x509 mandatory (no connection made without x509)
cert and key must be available for 1 & 2.
"""
self._x509_secure_connection = int(self._plugin_x509_configuration.get(u'x509_secured_requests',
self._default_x509_configuration[
u'x509_secured_requests']))
assert PanoptesValidators.valid_positive_integer(self._x509_secure_connection),\
u'x509 secure connection must be a positive integer'
assert self._x509_secure_connection < 3, u'x509 secure connection cannot be greater than 2'
cert_location = self._plugin_x509_configuration.get(u'x509_cert_location',
self._default_x509_configuration[u'x509_cert_location'])
assert PanoptesValidators.valid_nonempty_string(cert_location), u'cert location must be a valid string'
cert_filename = self._plugin_x509_configuration.get(u'x509_cert_filename',
self._default_x509_configuration[u'x509_cert_filename'])
assert PanoptesValidators.valid_nonempty_string(cert_filename), u'cert filename must be a valid string'
self._x509_certificate_file = u'{}/{}'.format(cert_location, cert_filename)
key_location = self._plugin_x509_configuration.get(u'x509_key_location',
self._default_x509_configuration[u'x509_key_location'])
assert PanoptesValidators.valid_nonempty_string(key_location), u'key location must be a valid string'
key_filename = self._plugin_x509_configuration.get(u'x509_key_filename',
self._default_x509_configuration[u'x509_key_filename'])
assert PanoptesValidators.valid_nonempty_string(key_filename), u'key filename must be a valid string'
self._x509_key_file = u'{}/{}'.format(key_location, key_filename)
def test_valid_positive_integer(self):
self.assertTrue(PanoptesValidators.valid_positive_integer(0))
self.assertTrue(
PanoptesValidators.valid_positive_integer(2**(2**(2**(2**2)))))
self.assertFalse(PanoptesValidators.valid_positive_integer(False))
self.assertFalse(PanoptesValidators.valid_positive_integer(True))
self.assertFalse(PanoptesValidators.valid_positive_integer(u""))
self.assertFalse(PanoptesValidators.valid_positive_integer(1.0))
self.assertFalse(PanoptesValidators.valid_positive_integer(-5))
self.assertTrue(PanoptesValidators.valid_positive_integer(-0))
def __init__(self, host, port, timeout, retries):
assert PanoptesValidators.valid_nonempty_string(host), u'host must a non-empty string'
assert PanoptesValidators.valid_port(port), u'port must be an integer between 1 and 65535'
assert PanoptesValidators.valid_nonzero_integer(timeout), u'timeout must be a integer greater than zero'
assert PanoptesValidators.valid_positive_integer(retries), u'retries must a non-negative integer'
self._host = host
self._port = port
self._timeout = timeout
self._retries = retries
self._easy_snmp_session = None
def _parse_snmp_configuration(self):
self._connection_factory_module = self._plugin_snmp_configuration.get(u'connection_factory_module',
self._default_snmp_configuration[
u'connection_factory_module'])
assert PanoptesValidators.valid_nonempty_string(
self._connection_factory_module), u'SNMP connection factory module must be a non-empty string'
self._connection_factory_class = self._plugin_snmp_configuration.get(u'connection_factory_class',
self._default_snmp_configuration[
u'connection_factory_class'])
assert PanoptesValidators.valid_nonempty_string(
self._connection_factory_class), u'SNMP connection factory class must be a non-empty string'
self._community_string_key = self._plugin_snmp_configuration.get(u'community_string_key',
self._default_snmp_configuration.get(
u'community_string_key'))
assert PanoptesValidators.valid_nonempty_string(
self._community_string_key), u'SNMP community string key must be a non-empty string'
self._port = int(
self._plugin_snmp_configuration.get(u'port', self._default_snmp_configuration[u'port']))
assert PanoptesValidators.valid_port(self._port), u'SNMP port must be a valid TCP/UDP port number'
self._proxy_port = int(
self._plugin_snmp_configuration.get(u'proxy_port', self._default_snmp_configuration[u'proxy_port']))
assert PanoptesValidators.valid_port(self._proxy_port), u'SNMP proxy port must be a valid TCP/UDP port number'
self._timeout = int(
self._plugin_snmp_configuration.get(u'timeout', self._default_snmp_configuration[u'timeout']))
assert PanoptesValidators.valid_nonzero_integer(
self._timeout), u'SNMP timeout must be a positive integer'
self._retries = int(
self._plugin_snmp_configuration.get(u'retries', self._default_snmp_configuration[u'retries']))
assert PanoptesValidators.valid_nonzero_integer(
self._retries), u'SNMP retries must be a positive integer'
self._non_repeaters = int(self._plugin_snmp_configuration.get(u'non_repeaters',
self._default_snmp_configuration[
u'non_repeaters']))
assert PanoptesValidators.valid_positive_integer(
self._non_repeaters), u'SNMP non-repeaters must be a positive integer'
self._max_repetitions = int(
self._plugin_snmp_configuration.get(u'max_repetitions',
self._default_snmp_configuration[u'max_repetitions']))
assert PanoptesValidators.valid_nonzero_integer(
self._max_repetitions), u'SNMP max-repetitions must be a positive integer'
def last_results(self, timestamp):
"""
Sets the last results time of the plugin
Args:
timestamp (int): The last results time of the plugin in Unix Epoch format
Returns:
None
"""
assert PanoptesValidators.valid_positive_integer(
timestamp), u"timestamp must be a positive integer."
try:
self.metadata_kv_store.set(
self.last_results_key,
str(timestamp),
expire=const.PLUGIN_AGENT_PLUGIN_TIMESTAMPS_EXPIRE)
except Exception as exp:
self.panoptes_context.logger.error(
u'Could not store value for last successful results time for plugin '
u'"%s": %s' % (self.name, str(exp)))
def __init__(self, context, path, timeout, retries=1, identifier=None):
"""
Creates and maintains state for a lock
Args:
path (str): A '/' separated path for the lock
timeout (int): in seconds. Must be a positive integer
retries (int): how many times to try before giving up. Zero implies try forever
identifier (str): Name to use for this lock contender. This can be useful for querying \
to see who the current lock contenders are
Returns:
PanoptesLock: lock
"""
assert PanoptesContextValidators.valid_panoptes_context(
context), u'context must be a valid PanoptesContext'
assert PanoptesValidators.valid_nonempty_string(path) and re.search(r"^/\S+", path), \
u'path must be a non-empty string that begins with /'
assert PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout must be a positive integer'
assert PanoptesValidators.valid_positive_integer(
retries), u'retries must be a non-negative integer'
assert PanoptesValidators.valid_nonempty_string(
identifier), u'identifier must be a non-empty string'
self._context = context
self._logger = self._context.logger
self._path = path
self._timeout = timeout
self._retries = retries
self._identifier = identifier
self._lock = None
self._locked = False
self._calling_module = get_calling_module_name(3)
self._get_lock()
def get_lock(self,
path,
timeout,
retries=1,
identifier=None,
listener=None):
"""
A wrapper around the kazoo library lock
Args:
path (str): A '/' separated path for the lock
timeout (int): in seconds. Must be a positive integer
retries (int): how many times to try before giving up. Zero implies try forever
identifier (str): Name to use for this lock contender. This can be useful for querying \
to see who the current lock contenders are
listener (callable): The callable to use to handle Zookeeper state changes
Returns:
kazoo.recipe.lock.Lock: lock
"""
assert PanoptesValidators.valid_nonempty_string(path) and re.search("^/\S+", path), \
u'path must be a non-empty string that begins with /'
assert PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout must be a positive integer'
assert PanoptesValidators.valid_positive_integer(
retries), u'retries must be a non-negative integer'
assert PanoptesValidators.valid_nonempty_string(
identifier), u'identifier must be a non-empty string'
assert (
not listener) or callable(listener), u'listener must be a callable'
logger = self.logger
calling_module = get_calling_module_name(2)
logger.info(u"Creating lock for module: " + calling_module +
u" with lock parameters: "
u"path=" + path + u",timeout=" + str(timeout) +
u",retries=" + str(retries) + u","
u"identifier=" + identifier)
try:
lock = self.zookeeper_client.Lock(path, identifier)
except Exception as e:
logger.error(u'Failed to create lock object: %s' % str(e))
return None
if retries == 0:
while True:
logger.info(
u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. '
% (identifier, path, lock.contenders()))
try:
lock.acquire(timeout=timeout)
except LockTimeout:
logger.info(
u'Timed out after %d seconds trying to acquire lock. Retrying.'
% timeout)
except Exception as e:
logger.info(u'Error in acquiring lock: %s. Retrying.' %
str(e))
if lock.is_acquired:
break
else:
tries = 0
while tries < retries:
logger.info(
u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. '
% (identifier, path, lock.contenders()))
try:
lock.acquire(timeout=timeout)
except LockTimeout:
logger.info(
u'Timed out after %d seconds trying to acquire lock. Retrying %d more times'
% (timeout, retries - tries))
except Exception as e:
logger.info(
u'Error in acquiring lock: %s. Retrying %d more times'
% (str(e), (retries - tries)))
if lock.is_acquired:
break
tries += 1
if not lock.is_acquired:
logger.warn(u'Unable to acquire lock after %d tries' % retries)
if lock.is_acquired:
logger.info(u'Lock acquired. Other contenders: %s' %
lock.contenders())
if listener:
self.zookeeper_client.add_listener(listener)
return lock
def __init__(self,
panoptes_context,
consumer_type,
topics,
client_id,
group,
keys,
poll_timeout,
callback,
validate=False,
session_timeout=60,
max_poll_records=500,
max_partition_fetch_bytes=1048576):
"""
This class implements a helper/wrapper for writing Panoptes Consumers
The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant
topics (which are combination of the site name and consumer type), filters records by the provided keys, checks
validity of the records by converting them to JSON and validating against consumer type specific schemas and
then calls the callback function provided for each record.
The consumer object also takes care of advancing the committed marker for each topic and partition - if the
callback fails or returns false, the marker is not advanced.
Args:
panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer
consumer_type (int): A valid consumer type
topics (list): A valid consumer type
client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \
logging by and group administration by Kafka
group (str): The Kafka group this consumer should be a part of
keys (list, None): A comma separated list of keys the consumer should filter against
poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \
no records are available on the Kafka bus
callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \
object) should be passed. The callable should return false if it cannot process the record due to
temporary \
issues - it would be redelivered to the callback in that case
validate (bool): Whether each incoming record should be validated. Defaults to False
session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \
Group Management system should consider the client disconnected
max_poll_records (int): The maximum number of records to fetch in one poll cycle
Returns:
None
"""
assert isinstance(
panoptes_context, PanoptesContext
), u'panoptes_context must be an instance of PanoptesContext'
assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \
u'PanoptesConsumerTypes'
assert PanoptesValidators.valid_nonempty_iterable_of_strings(
topics), u''
assert PanoptesValidators.valid_nonempty_string(
client_id), u'client_id must be a non-empty string'
assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \
u'keys must be None or a list of non-empty strings'
assert PanoptesValidators.valid_positive_integer(
poll_timeout), u'poll_timeout must be an integer'
assert hasattr(callback, u'__call__'), u'callback must be a callable'
assert isinstance(validate, bool), u'validate must be a boolean'
assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\
u'greater than zero'
assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\
u'greater than zero'
self._panoptes_context = panoptes_context
self._consumer_type = consumer_type
self._topics = topics
self._client_id = client_id
self._keys = keys
self._group = group
self._poll_timeout = poll_timeout * 1000
self._session_timeout = session_timeout * 1000
self._request_timeout = self._session_timeout * 3
self._heartbeat_interval = old_div(self._session_timeout, 3)
self._max_poll_records = max_poll_records
self._max_partition_fetch_bytes = max_partition_fetch_bytes
self._callback = callback
self._validate = validate
self._consumer = None
self._last_polled = 0
self._asked_to_stop = False