def _parse_snmp_configuration(self):
self._connection_factory_module = self._plugin_snmp_configuration.get(u'connection_factory_module',
self._default_snmp_configuration[
u'connection_factory_module'])
assert PanoptesValidators.valid_nonempty_string(
self._connection_factory_module), u'SNMP connection factory module must be a non-empty string'
self._connection_factory_class = self._plugin_snmp_configuration.get(u'connection_factory_class',
self._default_snmp_configuration[
u'connection_factory_class'])
assert PanoptesValidators.valid_nonempty_string(
self._connection_factory_class), u'SNMP connection factory class must be a non-empty string'
self._community_string_key = self._plugin_snmp_configuration.get(u'community_string_key',
self._default_snmp_configuration.get(
u'community_string_key'))
assert PanoptesValidators.valid_nonempty_string(
self._community_string_key), u'SNMP community string key must be a non-empty string'
self._port = int(
self._plugin_snmp_configuration.get(u'port', self._default_snmp_configuration[u'port']))
assert PanoptesValidators.valid_port(self._port), u'SNMP port must be a valid TCP/UDP port number'
self._proxy_port = int(
self._plugin_snmp_configuration.get(u'proxy_port', self._default_snmp_configuration[u'proxy_port']))
assert PanoptesValidators.valid_port(self._proxy_port), u'SNMP proxy port must be a valid TCP/UDP port number'
self._timeout = int(
self._plugin_snmp_configuration.get(u'timeout', self._default_snmp_configuration[u'timeout']))
assert PanoptesValidators.valid_nonzero_integer(
self._timeout), u'SNMP timeout must be a positive integer'
self._retries = int(
self._plugin_snmp_configuration.get(u'retries', self._default_snmp_configuration[u'retries']))
assert PanoptesValidators.valid_nonzero_integer(
self._retries), u'SNMP retries must be a positive integer'
self._non_repeaters = int(self._plugin_snmp_configuration.get(u'non_repeaters',
self._default_snmp_configuration[
u'non_repeaters']))
assert PanoptesValidators.valid_positive_integer(
self._non_repeaters), u'SNMP non-repeaters must be a positive integer'
self._max_repetitions = int(
self._plugin_snmp_configuration.get(u'max_repetitions',
self._default_snmp_configuration[u'max_repetitions']))
assert PanoptesValidators.valid_nonzero_integer(
self._max_repetitions), u'SNMP max-repetitions must be a positive integer'
def __init__(self, resource_site, resource_class, resource_subclass, resource_type,
resource_id, resource_endpoint, resource_creation_timestamp=None,
resource_plugin=None, resource_ttl=const.RESOURCE_MANAGER_RESOURCE_EXPIRE):
assert PanoptesValidators.valid_nonempty_string(resource_site), u'resource_site must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(resource_class), u'resource_class must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(resource_subclass), u'resource_subclass must be a non-empty'
assert PanoptesValidators.valid_nonempty_string(resource_type), u'resource_type must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(resource_id), u'resource_id must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(resource_endpoint), u'resource_endpoint must be a non-empty str'
assert PanoptesValidators.valid_none_or_nonempty_string(
resource_plugin), u'resource_plugin must be None or a non-empty str'
assert PanoptesValidators.valid_nonzero_integer(resource_ttl), u'resource_ttl must be an integer greater than 0'
self.__data = OrderedDict()
self.__data[u'resource_site'] = str(resource_site)
self.__data[u'resource_class'] = str(resource_class)
self.__data[u'resource_subclass'] = str(resource_subclass)
self.__data[u'resource_type'] = str(resource_type)
self.__data[u'resource_id'] = str(resource_id)
self.__data[u'resource_endpoint'] = str(resource_endpoint)
self.__data[u'resource_metadata'] = OrderedDict()
if not resource_creation_timestamp:
self.__data[u'resource_creation_timestamp'] = time()
else:
self.__data[u'resource_creation_timestamp'] = resource_creation_timestamp
self.__data[u'resource_plugin'] = resource_plugin
self.__data[u'resource_metadata'][u'_resource_ttl'] = str(resource_ttl)
def getset(self, key, value, expire=604800):
"""
Set the value associated with the key in the key/value store and returns the previously set value
This does an 'upsert' - inserts the key/value if it does not exist and updates the value if the key exists
Args:
key (str): The key whose value should be set
value (str): The value to set
expire (int): A positive integer that, if sets, would expire the key in the number of seconds specified
Returns:
None: Nothing. Passes through exceptions in case of failure
"""
assert PanoptesValidators.valid_nonempty_string(
key), u'key must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(
value), u'key value be a non-empty str'
assert PanoptesValidators.valid_nonzero_integer(
expire), u'expire must be an integer greater than zero'
value = self._get_redis_shard(key).getset(self._normalized_key(key),
value.encode('utf-8'))
self._get_redis_shard(key).expire(self._normalized_key(key), expire)
if value is not None:
return value.decode('utf-8')
def __init__(self, panoptes_context, plugin_type, plugin_type_display_name, celery_config, lock_timeout,
plugin_scheduler_task, plugin_subtype=None):
assert PanoptesContextValidators.valid_panoptes_context(
panoptes_context), u'panoptes_context must be an instance of PanoptesContext'
assert PanoptesValidators.valid_nonempty_string(plugin_type), u'plugin_type must be a non-empty str'
assert PanoptesValidators.valid_nonempty_string(plugin_type_display_name), \
u'plugin_type_display_name must be a non-empty str'
assert PanoptesCeleryValidators.valid_celery_config(
celery_config), u'celery_config must be an instance of PanoptesCeleryConfig'
assert PanoptesValidators.valid_nonzero_integer(lock_timeout), u'lock_timeout must be an int greater than zero'
assert PanoptesValidators.valid_callback(plugin_scheduler_task), u'plugin_scheduler_task must be a callable'
assert plugin_type is None or PanoptesValidators.valid_nonempty_string(plugin_type), u'plugin_type must be a ' \
u'None or a non-empty str'
self._panoptes_context = panoptes_context
self._config = self._panoptes_context.config_dict
self._logger = self._panoptes_context.logger
self._shutdown_plugin_scheduler = threading.Event()
self._plugin_scheduler_celery_beat_service = None
self._celery_config = celery_config
self._celery = None
self._t = None
self._lock = None
self._plugin_type = plugin_type
self._plugin_subtype = plugin_subtype
self._plugin_type_display_name = plugin_type_display_name
self._lock_timeout = lock_timeout
self._plugin_scheduler_task = plugin_scheduler_task
self._tour_of_duty = PanoptesTourOfDuty(splay_percent=50)
self._cycles_without_lock = 0
def __init__(self,
resource,
group_type,
interval,
creation_timestamp=None):
assert PanoptesMetricValidators.valid_panoptes_resource(
resource), u'resource must be an instance of PanoptesResource'
assert PanoptesValidators.valid_nonempty_string(
group_type), u'group_type must be a non-empty string'
assert PanoptesValidators.valid_nonzero_integer(
interval), u'interval must a integer greater than zero'
self.__data = dict()
self.__metrics_index = {
metric_type: list()
for metric_type in METRIC_TYPE_NAMES
}
self.__data[u'metrics_group_type'] = group_type
self.__data[u'metrics_group_interval'] = interval
self.__data[u'metrics_group_creation_timestamp'] = round(time(), METRICS_TIMESTAMP_PRECISION) \
if creation_timestamp is None else creation_timestamp
self.__data[
u'metrics_group_schema_version'] = METRICS_GROUP_SCHEMA_VERSION
self.__data[u'resource'] = resource
self.__data[u'metrics'] = set()
self.__data[u'dimensions'] = set()
self._data_lock = threading.Lock()
def test_valid_nonzero_integer(self):
self.assertFalse(PanoptesValidators.valid_nonzero_integer(-1))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(0))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(-0))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(1.0))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(u""))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(False))
self.assertFalse(PanoptesValidators.valid_nonzero_integer(True))
self.assertTrue(
PanoptesValidators.valid_nonzero_integer(2**(2**(2**(2**2)))))
def __init__(self, host, port, timeout, retries):
assert PanoptesValidators.valid_nonempty_string(host), u'host must a non-empty string'
assert PanoptesValidators.valid_port(port), u'port must be an integer between 1 and 65535'
assert PanoptesValidators.valid_nonzero_integer(timeout), u'timeout must be a integer greater than zero'
assert PanoptesValidators.valid_positive_integer(retries), u'retries must a non-negative integer'
self._host = host
self._port = port
self._timeout = timeout
self._retries = retries
self._easy_snmp_session = None
def __init__(self, count=10, timeout=10, hostname='localhost'):
assert PanoptesValidators.valid_nonzero_integer(count), 'count must be integer > 0'
assert PanoptesValidators.valid_nonempty_string(hostname), 'hostname must be nonempty string'
assert PanoptesValidators.valid_nonzero_integer(timeout), 'timeout must be integer > 0'
super(PanoptesPingDirect, self).__init__()
try:
resp = subprocess.check_output(
['/bin/ping', '-c', str(count), '-w', str(timeout), hostname],
stderr=subprocess.STDOUT,
universal_newlines=True # return string not bytes
)
self._get_ping_stats(resp)
except subprocess.CalledProcessError as e:
self._get_ping_stats(e.output)
if self._response['packets_transmitted'] is not None:
raise PanoptesPingTimeoutException("Ping timed out with response: " + str(self._response))
raise PanoptesPingException(e.output)
except Exception as e:
raise PanoptesPingException(str(e))
def __init__(self,
tasks=1000,
seconds=14400,
memory_growth_mb=200,
splay_percent=0):
assert PanoptesValidators.valid_nonzero_integer(
tasks), u'tasks must a integer greater than zero'
assert PanoptesValidators.valid_nonzero_integer(
seconds), u'seconds must a integer greater than zero'
assert PanoptesValidators.valid_nonzero_integer(
memory_growth_mb
), u'memory_growth_mb must a integer greater than zero'
assert PanoptesValidators.valid_number(
splay_percent), u'splay_percent must a number'
assert 0 <= splay_percent <= 100, u'splay_percent must be a number between 0 and 100, inclusive'
self._start_time = time.time()
if platform.system() == u'Linux':
# On Linux platforms, memory is reported in kilobytes
self._memory_divider = 1024
else:
# On all other platforms, assume bytes (true for Mac OS X)
self._memory_divider = 1048576
self._initial_memory_mb = self._get_memory_utilization_in_mb()
self._task_count = 0
self._tasks = tasks
self._seconds = seconds
self._memory_growth_mb = memory_growth_mb
self._splay_percent = float(splay_percent) / 100
self._adjusted_tasks = round(
self._tasks * (1 + (random.random() * self._splay_percent)))
self._adjusted_seconds = round(
self._seconds * (1 + (random.random() * self._splay_percent)))
self._adjusted_memory_growth_mb = round(
self._memory_growth_mb * (1 +
(random.random() * self._splay_percent)))
def __init__(self, count=10, timeout=10, hostname=u'localhost'):
assert PanoptesValidators.valid_nonzero_integer(
count), u'count must be integer > 0'
assert PanoptesValidators.valid_nonempty_string(
hostname), u'hostname must be nonempty string'
assert PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout must be integer > 0'
self._response = dict() # dictionary containing ping statistics
self._response[u'packets_transmitted'] = None
self._response[u'packets_received'] = None
self._response[u'packet_loss_pct'] = None
self._response[u'execution_time'] = None
self._response[u'round_trip_min'] = None
self._response[u'round_trip_avg'] = None
self._response[u'round_trip_max'] = None
self._response[u'round_trip_stddev'] = None
try:
resp = subprocess.check_output(
[
u'/bin/ping', u'-c',
str(count), u'-w',
str(timeout), hostname
],
stderr=subprocess.STDOUT,
universal_newlines=True # return string not bytes
)
self._get_ping_stats(resp)
except subprocess.CalledProcessError as e:
self._get_ping_stats(e.output)
if self._response[u'packets_transmitted'] is not None:
raise PanoptesPingTimeoutException(
u"Ping timed out with response: " + str(self._response))
raise PanoptesPingException(e.output)
except Exception as e:
raise PanoptesPingException(e.message)
def __init__(self, context, path, timeout, retries=1, identifier=None):
"""
Creates and maintains state for a lock
Args:
path (str): A '/' separated path for the lock
timeout (int): in seconds. Must be a positive integer
retries (int): how many times to try before giving up. Zero implies try forever
identifier (str): Name to use for this lock contender. This can be useful for querying \
to see who the current lock contenders are
Returns:
PanoptesLock: lock
"""
assert PanoptesContextValidators.valid_panoptes_context(
context), u'context must be a valid PanoptesContext'
assert PanoptesValidators.valid_nonempty_string(path) and re.search(r"^/\S+", path), \
u'path must be a non-empty string that begins with /'
assert PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout must be a positive integer'
assert PanoptesValidators.valid_positive_integer(
retries), u'retries must be a non-negative integer'
assert PanoptesValidators.valid_nonempty_string(
identifier), u'identifier must be a non-empty string'
self._context = context
self._logger = self._context.logger
self._path = path
self._timeout = timeout
self._retries = retries
self._identifier = identifier
self._lock = None
self._locked = False
self._calling_module = get_calling_module_name(3)
self._get_lock()
def get_lock(self,
path,
timeout,
retries=1,
identifier=None,
listener=None):
"""
A wrapper around the kazoo library lock
Args:
path (str): A '/' separated path for the lock
timeout (int): in seconds. Must be a positive integer
retries (int): how many times to try before giving up. Zero implies try forever
identifier (str): Name to use for this lock contender. This can be useful for querying \
to see who the current lock contenders are
listener (callable): The callable to use to handle Zookeeper state changes
Returns:
kazoo.recipe.lock.Lock: lock
"""
assert PanoptesValidators.valid_nonempty_string(path) and re.search("^/\S+", path), \
u'path must be a non-empty string that begins with /'
assert PanoptesValidators.valid_nonzero_integer(
timeout), u'timeout must be a positive integer'
assert PanoptesValidators.valid_positive_integer(
retries), u'retries must be a non-negative integer'
assert PanoptesValidators.valid_nonempty_string(
identifier), u'identifier must be a non-empty string'
assert (
not listener) or callable(listener), u'listener must be a callable'
logger = self.logger
calling_module = get_calling_module_name(2)
logger.info(u"Creating lock for module: " + calling_module +
u" with lock parameters: "
u"path=" + path + u",timeout=" + str(timeout) +
u",retries=" + str(retries) + u","
u"identifier=" + identifier)
try:
lock = self.zookeeper_client.Lock(path, identifier)
except Exception as e:
logger.error(u'Failed to create lock object: %s' % str(e))
return None
if retries == 0:
while True:
logger.info(
u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. '
% (identifier, path, lock.contenders()))
try:
lock.acquire(timeout=timeout)
except LockTimeout:
logger.info(
u'Timed out after %d seconds trying to acquire lock. Retrying.'
% timeout)
except Exception as e:
logger.info(u'Error in acquiring lock: %s. Retrying.' %
str(e))
if lock.is_acquired:
break
else:
tries = 0
while tries < retries:
logger.info(
u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. '
% (identifier, path, lock.contenders()))
try:
lock.acquire(timeout=timeout)
except LockTimeout:
logger.info(
u'Timed out after %d seconds trying to acquire lock. Retrying %d more times'
% (timeout, retries - tries))
except Exception as e:
logger.info(
u'Error in acquiring lock: %s. Retrying %d more times'
% (str(e), (retries - tries)))
if lock.is_acquired:
break
tries += 1
if not lock.is_acquired:
logger.warn(u'Unable to acquire lock after %d tries' % retries)
if lock.is_acquired:
logger.info(u'Lock acquired. Other contenders: %s' %
lock.contenders())
if listener:
self.zookeeper_client.add_listener(listener)
return lock
def __init__(self,
panoptes_context,
consumer_type,
topics,
client_id,
group,
keys,
poll_timeout,
callback,
validate=False,
session_timeout=60,
max_poll_records=500,
max_partition_fetch_bytes=1048576):
"""
This class implements a helper/wrapper for writing Panoptes Consumers
The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant
topics (which are combination of the site name and consumer type), filters records by the provided keys, checks
validity of the records by converting them to JSON and validating against consumer type specific schemas and
then calls the callback function provided for each record.
The consumer object also takes care of advancing the committed marker for each topic and partition - if the
callback fails or returns false, the marker is not advanced.
Args:
panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer
consumer_type (int): A valid consumer type
topics (list): A valid consumer type
client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \
logging by and group administration by Kafka
group (str): The Kafka group this consumer should be a part of
keys (list, None): A comma separated list of keys the consumer should filter against
poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \
no records are available on the Kafka bus
callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \
object) should be passed. The callable should return false if it cannot process the record due to
temporary \
issues - it would be redelivered to the callback in that case
validate (bool): Whether each incoming record should be validated. Defaults to False
session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \
Group Management system should consider the client disconnected
max_poll_records (int): The maximum number of records to fetch in one poll cycle
Returns:
None
"""
assert isinstance(
panoptes_context, PanoptesContext
), u'panoptes_context must be an instance of PanoptesContext'
assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \
u'PanoptesConsumerTypes'
assert PanoptesValidators.valid_nonempty_iterable_of_strings(
topics), u''
assert PanoptesValidators.valid_nonempty_string(
client_id), u'client_id must be a non-empty string'
assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \
u'keys must be None or a list of non-empty strings'
assert PanoptesValidators.valid_positive_integer(
poll_timeout), u'poll_timeout must be an integer'
assert hasattr(callback, u'__call__'), u'callback must be a callable'
assert isinstance(validate, bool), u'validate must be a boolean'
assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\
u'greater than zero'
assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\
u'greater than zero'
self._panoptes_context = panoptes_context
self._consumer_type = consumer_type
self._topics = topics
self._client_id = client_id
self._keys = keys
self._group = group
self._poll_timeout = poll_timeout * 1000
self._session_timeout = session_timeout * 1000
self._request_timeout = self._session_timeout * 3
self._heartbeat_interval = old_div(self._session_timeout, 3)
self._max_poll_records = max_poll_records
self._max_partition_fetch_bytes = max_partition_fetch_bytes
self._callback = callback
self._validate = validate
self._consumer = None
self._last_polled = 0
self._asked_to_stop = False