def bulk_walk(self, oid, non_repeaters=0, max_repetitions=10): """ Takes a single numeric oid and fetches oids under it's subtree using GETBULKs Args: oid (str): A numeric oid non_repeaters (int): A non-negative integer max_repetitions (int): A non-negative integer Returns: list(PanoptesSNMPVariable): A list of PanoptesSNMPVariables within the subtree of the provided oid """ assert PanoptesValidators.valid_numeric_snmp_oid(oid), u'oid must be numeric string with a leading period' assert PanoptesValidators.valid_positive_integer(non_repeaters), u'non_repeaters must a positive integer' assert PanoptesValidators.valid_positive_integer(max_repetitions), u'max_repetitions must a positive integer' assert self._easy_snmp_session is not None, u'PanoptesSNMPSession not initialized' base_oid = oid varbinds = list() running = 1 while running: results = self.get_bulk(oid=oid, non_repeaters=non_repeaters, max_repetitions=max_repetitions, base_oid=base_oid) varbinds.extend(results) if len(results) < max_repetitions: break oid = base_oid + u'.' + results[-1].index return varbinds
def get_bulk(self, oid, non_repeaters=0, max_repetitions=10, base_oid=None): """ Takes a single numeric oid and returns a list of PanoptesSNMPVariables which start with the oid Args: oid (str): A numeric oid non_repeaters (int): A non-negative integer max_repetitions (int): A non-negative integer base_oid (str): An optional numeric oid which is used to set the oid in the returned PanoptesSNMPVariables Returns: list(): A list of PanoptesSNMPVariables starting with the provided oid """ assert PanoptesValidators.valid_numeric_snmp_oid(oid), u'oid must be numeric string with a leading period' assert PanoptesValidators.valid_positive_integer(non_repeaters), u'non_repeaters must a positive integer' assert PanoptesValidators.valid_positive_integer(max_repetitions), u'max_repetitions must a positive integer' assert (base_oid is None) or PanoptesValidators.valid_numeric_snmp_oid( base_oid), u'oid must be numeric string with a leading period' assert self._easy_snmp_session is not None, u'PanoptesSNMPSession not initialized' try: varbinds = self._easy_snmp_session.get_bulk(oids=oid, non_repeaters=non_repeaters, max_repetitions=max_repetitions) result = list() for varbind in varbinds: if varbind.oid.startswith(base_oid): result.append(PanoptesSNMPVariable(queried_oid=base_oid if base_oid else oid, oid=varbind.oid, index=varbind.oid_index, value=varbind.value, snmp_type=varbind.snmp_type)) return result except Exception as e: raise self._wrap_panoptes_snmp_exception(e)
def _parse_x509_configuration(self): """ Parses the config spec to provide some details on the x509 communication. This is effectively requests over x509 based on the value of _x509_secure_connection. 0 - x509 communications 1 - x509 where available (optional) 2 - x509 mandatory (no connection made without x509) cert and key must be available for 1 & 2. """ self._x509_secure_connection = int(self._plugin_x509_configuration.get(u'x509_secured_requests', self._default_x509_configuration[ u'x509_secured_requests'])) assert PanoptesValidators.valid_positive_integer(self._x509_secure_connection),\ u'x509 secure connection must be a positive integer' assert self._x509_secure_connection < 3, u'x509 secure connection cannot be greater than 2' cert_location = self._plugin_x509_configuration.get(u'x509_cert_location', self._default_x509_configuration[u'x509_cert_location']) assert PanoptesValidators.valid_nonempty_string(cert_location), u'cert location must be a valid string' cert_filename = self._plugin_x509_configuration.get(u'x509_cert_filename', self._default_x509_configuration[u'x509_cert_filename']) assert PanoptesValidators.valid_nonempty_string(cert_filename), u'cert filename must be a valid string' self._x509_certificate_file = u'{}/{}'.format(cert_location, cert_filename) key_location = self._plugin_x509_configuration.get(u'x509_key_location', self._default_x509_configuration[u'x509_key_location']) assert PanoptesValidators.valid_nonempty_string(key_location), u'key location must be a valid string' key_filename = self._plugin_x509_configuration.get(u'x509_key_filename', self._default_x509_configuration[u'x509_key_filename']) assert PanoptesValidators.valid_nonempty_string(key_filename), u'key filename must be a valid string' self._x509_key_file = u'{}/{}'.format(key_location, key_filename)
def test_valid_positive_integer(self): self.assertTrue(PanoptesValidators.valid_positive_integer(0)) self.assertTrue( PanoptesValidators.valid_positive_integer(2**(2**(2**(2**2))))) self.assertFalse(PanoptesValidators.valid_positive_integer(False)) self.assertFalse(PanoptesValidators.valid_positive_integer(True)) self.assertFalse(PanoptesValidators.valid_positive_integer(u"")) self.assertFalse(PanoptesValidators.valid_positive_integer(1.0)) self.assertFalse(PanoptesValidators.valid_positive_integer(-5)) self.assertTrue(PanoptesValidators.valid_positive_integer(-0))
def __init__(self, host, port, timeout, retries): assert PanoptesValidators.valid_nonempty_string(host), u'host must a non-empty string' assert PanoptesValidators.valid_port(port), u'port must be an integer between 1 and 65535' assert PanoptesValidators.valid_nonzero_integer(timeout), u'timeout must be a integer greater than zero' assert PanoptesValidators.valid_positive_integer(retries), u'retries must a non-negative integer' self._host = host self._port = port self._timeout = timeout self._retries = retries self._easy_snmp_session = None
def _parse_snmp_configuration(self): self._connection_factory_module = self._plugin_snmp_configuration.get(u'connection_factory_module', self._default_snmp_configuration[ u'connection_factory_module']) assert PanoptesValidators.valid_nonempty_string( self._connection_factory_module), u'SNMP connection factory module must be a non-empty string' self._connection_factory_class = self._plugin_snmp_configuration.get(u'connection_factory_class', self._default_snmp_configuration[ u'connection_factory_class']) assert PanoptesValidators.valid_nonempty_string( self._connection_factory_class), u'SNMP connection factory class must be a non-empty string' self._community_string_key = self._plugin_snmp_configuration.get(u'community_string_key', self._default_snmp_configuration.get( u'community_string_key')) assert PanoptesValidators.valid_nonempty_string( self._community_string_key), u'SNMP community string key must be a non-empty string' self._port = int( self._plugin_snmp_configuration.get(u'port', self._default_snmp_configuration[u'port'])) assert PanoptesValidators.valid_port(self._port), u'SNMP port must be a valid TCP/UDP port number' self._proxy_port = int( self._plugin_snmp_configuration.get(u'proxy_port', self._default_snmp_configuration[u'proxy_port'])) assert PanoptesValidators.valid_port(self._proxy_port), u'SNMP proxy port must be a valid TCP/UDP port number' self._timeout = int( self._plugin_snmp_configuration.get(u'timeout', self._default_snmp_configuration[u'timeout'])) assert PanoptesValidators.valid_nonzero_integer( self._timeout), u'SNMP timeout must be a positive integer' self._retries = int( self._plugin_snmp_configuration.get(u'retries', self._default_snmp_configuration[u'retries'])) assert PanoptesValidators.valid_nonzero_integer( self._retries), u'SNMP retries must be a positive integer' self._non_repeaters = int(self._plugin_snmp_configuration.get(u'non_repeaters', self._default_snmp_configuration[ u'non_repeaters'])) assert PanoptesValidators.valid_positive_integer( self._non_repeaters), u'SNMP non-repeaters must be a positive integer' self._max_repetitions = int( self._plugin_snmp_configuration.get(u'max_repetitions', self._default_snmp_configuration[u'max_repetitions'])) assert PanoptesValidators.valid_nonzero_integer( self._max_repetitions), u'SNMP max-repetitions must be a positive integer'
def last_results(self, timestamp): """ Sets the last results time of the plugin Args: timestamp (int): The last results time of the plugin in Unix Epoch format Returns: None """ assert PanoptesValidators.valid_positive_integer( timestamp), u"timestamp must be a positive integer." try: self.metadata_kv_store.set( self.last_results_key, str(timestamp), expire=const.PLUGIN_AGENT_PLUGIN_TIMESTAMPS_EXPIRE) except Exception as exp: self.panoptes_context.logger.error( u'Could not store value for last successful results time for plugin ' u'"%s": %s' % (self.name, str(exp)))
def __init__(self, context, path, timeout, retries=1, identifier=None): """ Creates and maintains state for a lock Args: path (str): A '/' separated path for the lock timeout (int): in seconds. Must be a positive integer retries (int): how many times to try before giving up. Zero implies try forever identifier (str): Name to use for this lock contender. This can be useful for querying \ to see who the current lock contenders are Returns: PanoptesLock: lock """ assert PanoptesContextValidators.valid_panoptes_context( context), u'context must be a valid PanoptesContext' assert PanoptesValidators.valid_nonempty_string(path) and re.search(r"^/\S+", path), \ u'path must be a non-empty string that begins with /' assert PanoptesValidators.valid_nonzero_integer( timeout), u'timeout must be a positive integer' assert PanoptesValidators.valid_positive_integer( retries), u'retries must be a non-negative integer' assert PanoptesValidators.valid_nonempty_string( identifier), u'identifier must be a non-empty string' self._context = context self._logger = self._context.logger self._path = path self._timeout = timeout self._retries = retries self._identifier = identifier self._lock = None self._locked = False self._calling_module = get_calling_module_name(3) self._get_lock()
def get_lock(self, path, timeout, retries=1, identifier=None, listener=None): """ A wrapper around the kazoo library lock Args: path (str): A '/' separated path for the lock timeout (int): in seconds. Must be a positive integer retries (int): how many times to try before giving up. Zero implies try forever identifier (str): Name to use for this lock contender. This can be useful for querying \ to see who the current lock contenders are listener (callable): The callable to use to handle Zookeeper state changes Returns: kazoo.recipe.lock.Lock: lock """ assert PanoptesValidators.valid_nonempty_string(path) and re.search("^/\S+", path), \ u'path must be a non-empty string that begins with /' assert PanoptesValidators.valid_nonzero_integer( timeout), u'timeout must be a positive integer' assert PanoptesValidators.valid_positive_integer( retries), u'retries must be a non-negative integer' assert PanoptesValidators.valid_nonempty_string( identifier), u'identifier must be a non-empty string' assert ( not listener) or callable(listener), u'listener must be a callable' logger = self.logger calling_module = get_calling_module_name(2) logger.info(u"Creating lock for module: " + calling_module + u" with lock parameters: " u"path=" + path + u",timeout=" + str(timeout) + u",retries=" + str(retries) + u"," u"identifier=" + identifier) try: lock = self.zookeeper_client.Lock(path, identifier) except Exception as e: logger.error(u'Failed to create lock object: %s' % str(e)) return None if retries == 0: while True: logger.info( u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. ' % (identifier, path, lock.contenders())) try: lock.acquire(timeout=timeout) except LockTimeout: logger.info( u'Timed out after %d seconds trying to acquire lock. Retrying.' % timeout) except Exception as e: logger.info(u'Error in acquiring lock: %s. Retrying.' % str(e)) if lock.is_acquired: break else: tries = 0 while tries < retries: logger.info( u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. ' % (identifier, path, lock.contenders())) try: lock.acquire(timeout=timeout) except LockTimeout: logger.info( u'Timed out after %d seconds trying to acquire lock. Retrying %d more times' % (timeout, retries - tries)) except Exception as e: logger.info( u'Error in acquiring lock: %s. Retrying %d more times' % (str(e), (retries - tries))) if lock.is_acquired: break tries += 1 if not lock.is_acquired: logger.warn(u'Unable to acquire lock after %d tries' % retries) if lock.is_acquired: logger.info(u'Lock acquired. Other contenders: %s' % lock.contenders()) if listener: self.zookeeper_client.add_listener(listener) return lock
def __init__(self, panoptes_context, consumer_type, topics, client_id, group, keys, poll_timeout, callback, validate=False, session_timeout=60, max_poll_records=500, max_partition_fetch_bytes=1048576): """ This class implements a helper/wrapper for writing Panoptes Consumers The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant topics (which are combination of the site name and consumer type), filters records by the provided keys, checks validity of the records by converting them to JSON and validating against consumer type specific schemas and then calls the callback function provided for each record. The consumer object also takes care of advancing the committed marker for each topic and partition - if the callback fails or returns false, the marker is not advanced. Args: panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer consumer_type (int): A valid consumer type topics (list): A valid consumer type client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \ logging by and group administration by Kafka group (str): The Kafka group this consumer should be a part of keys (list, None): A comma separated list of keys the consumer should filter against poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \ no records are available on the Kafka bus callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \ object) should be passed. The callable should return false if it cannot process the record due to temporary \ issues - it would be redelivered to the callback in that case validate (bool): Whether each incoming record should be validated. Defaults to False session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \ Group Management system should consider the client disconnected max_poll_records (int): The maximum number of records to fetch in one poll cycle Returns: None """ assert isinstance( panoptes_context, PanoptesContext ), u'panoptes_context must be an instance of PanoptesContext' assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \ u'PanoptesConsumerTypes' assert PanoptesValidators.valid_nonempty_iterable_of_strings( topics), u'' assert PanoptesValidators.valid_nonempty_string( client_id), u'client_id must be a non-empty string' assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \ u'keys must be None or a list of non-empty strings' assert PanoptesValidators.valid_positive_integer( poll_timeout), u'poll_timeout must be an integer' assert hasattr(callback, u'__call__'), u'callback must be a callable' assert isinstance(validate, bool), u'validate must be a boolean' assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\ u'greater than zero' assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\ u'greater than zero' self._panoptes_context = panoptes_context self._consumer_type = consumer_type self._topics = topics self._client_id = client_id self._keys = keys self._group = group self._poll_timeout = poll_timeout * 1000 self._session_timeout = session_timeout * 1000 self._request_timeout = self._session_timeout * 3 self._heartbeat_interval = old_div(self._session_timeout, 3) self._max_poll_records = max_poll_records self._max_partition_fetch_bytes = max_partition_fetch_bytes self._callback = callback self._validate = validate self._consumer = None self._last_polled = 0 self._asked_to_stop = False