def _parse_snmp_configuration(self): self._connection_factory_module = self._plugin_snmp_configuration.get(u'connection_factory_module', self._default_snmp_configuration[ u'connection_factory_module']) assert PanoptesValidators.valid_nonempty_string( self._connection_factory_module), u'SNMP connection factory module must be a non-empty string' self._connection_factory_class = self._plugin_snmp_configuration.get(u'connection_factory_class', self._default_snmp_configuration[ u'connection_factory_class']) assert PanoptesValidators.valid_nonempty_string( self._connection_factory_class), u'SNMP connection factory class must be a non-empty string' self._community_string_key = self._plugin_snmp_configuration.get(u'community_string_key', self._default_snmp_configuration.get( u'community_string_key')) assert PanoptesValidators.valid_nonempty_string( self._community_string_key), u'SNMP community string key must be a non-empty string' self._port = int( self._plugin_snmp_configuration.get(u'port', self._default_snmp_configuration[u'port'])) assert PanoptesValidators.valid_port(self._port), u'SNMP port must be a valid TCP/UDP port number' self._proxy_port = int( self._plugin_snmp_configuration.get(u'proxy_port', self._default_snmp_configuration[u'proxy_port'])) assert PanoptesValidators.valid_port(self._proxy_port), u'SNMP proxy port must be a valid TCP/UDP port number' self._timeout = int( self._plugin_snmp_configuration.get(u'timeout', self._default_snmp_configuration[u'timeout'])) assert PanoptesValidators.valid_nonzero_integer( self._timeout), u'SNMP timeout must be a positive integer' self._retries = int( self._plugin_snmp_configuration.get(u'retries', self._default_snmp_configuration[u'retries'])) assert PanoptesValidators.valid_nonzero_integer( self._retries), u'SNMP retries must be a positive integer' self._non_repeaters = int(self._plugin_snmp_configuration.get(u'non_repeaters', self._default_snmp_configuration[ u'non_repeaters'])) assert PanoptesValidators.valid_positive_integer( self._non_repeaters), u'SNMP non-repeaters must be a positive integer' self._max_repetitions = int( self._plugin_snmp_configuration.get(u'max_repetitions', self._default_snmp_configuration[u'max_repetitions'])) assert PanoptesValidators.valid_nonzero_integer( self._max_repetitions), u'SNMP max-repetitions must be a positive integer'
def __init__(self, resource_site, resource_class, resource_subclass, resource_type, resource_id, resource_endpoint, resource_creation_timestamp=None, resource_plugin=None, resource_ttl=const.RESOURCE_MANAGER_RESOURCE_EXPIRE): assert PanoptesValidators.valid_nonempty_string(resource_site), u'resource_site must be a non-empty str' assert PanoptesValidators.valid_nonempty_string(resource_class), u'resource_class must be a non-empty str' assert PanoptesValidators.valid_nonempty_string(resource_subclass), u'resource_subclass must be a non-empty' assert PanoptesValidators.valid_nonempty_string(resource_type), u'resource_type must be a non-empty str' assert PanoptesValidators.valid_nonempty_string(resource_id), u'resource_id must be a non-empty str' assert PanoptesValidators.valid_nonempty_string(resource_endpoint), u'resource_endpoint must be a non-empty str' assert PanoptesValidators.valid_none_or_nonempty_string( resource_plugin), u'resource_plugin must be None or a non-empty str' assert PanoptesValidators.valid_nonzero_integer(resource_ttl), u'resource_ttl must be an integer greater than 0' self.__data = OrderedDict() self.__data[u'resource_site'] = str(resource_site) self.__data[u'resource_class'] = str(resource_class) self.__data[u'resource_subclass'] = str(resource_subclass) self.__data[u'resource_type'] = str(resource_type) self.__data[u'resource_id'] = str(resource_id) self.__data[u'resource_endpoint'] = str(resource_endpoint) self.__data[u'resource_metadata'] = OrderedDict() if not resource_creation_timestamp: self.__data[u'resource_creation_timestamp'] = time() else: self.__data[u'resource_creation_timestamp'] = resource_creation_timestamp self.__data[u'resource_plugin'] = resource_plugin self.__data[u'resource_metadata'][u'_resource_ttl'] = str(resource_ttl)
def getset(self, key, value, expire=604800): """ Set the value associated with the key in the key/value store and returns the previously set value This does an 'upsert' - inserts the key/value if it does not exist and updates the value if the key exists Args: key (str): The key whose value should be set value (str): The value to set expire (int): A positive integer that, if sets, would expire the key in the number of seconds specified Returns: None: Nothing. Passes through exceptions in case of failure """ assert PanoptesValidators.valid_nonempty_string( key), u'key must be a non-empty str' assert PanoptesValidators.valid_nonempty_string( value), u'key value be a non-empty str' assert PanoptesValidators.valid_nonzero_integer( expire), u'expire must be an integer greater than zero' value = self._get_redis_shard(key).getset(self._normalized_key(key), value.encode('utf-8')) self._get_redis_shard(key).expire(self._normalized_key(key), expire) if value is not None: return value.decode('utf-8')
def __init__(self, panoptes_context, plugin_type, plugin_type_display_name, celery_config, lock_timeout, plugin_scheduler_task, plugin_subtype=None): assert PanoptesContextValidators.valid_panoptes_context( panoptes_context), u'panoptes_context must be an instance of PanoptesContext' assert PanoptesValidators.valid_nonempty_string(plugin_type), u'plugin_type must be a non-empty str' assert PanoptesValidators.valid_nonempty_string(plugin_type_display_name), \ u'plugin_type_display_name must be a non-empty str' assert PanoptesCeleryValidators.valid_celery_config( celery_config), u'celery_config must be an instance of PanoptesCeleryConfig' assert PanoptesValidators.valid_nonzero_integer(lock_timeout), u'lock_timeout must be an int greater than zero' assert PanoptesValidators.valid_callback(plugin_scheduler_task), u'plugin_scheduler_task must be a callable' assert plugin_type is None or PanoptesValidators.valid_nonempty_string(plugin_type), u'plugin_type must be a ' \ u'None or a non-empty str' self._panoptes_context = panoptes_context self._config = self._panoptes_context.config_dict self._logger = self._panoptes_context.logger self._shutdown_plugin_scheduler = threading.Event() self._plugin_scheduler_celery_beat_service = None self._celery_config = celery_config self._celery = None self._t = None self._lock = None self._plugin_type = plugin_type self._plugin_subtype = plugin_subtype self._plugin_type_display_name = plugin_type_display_name self._lock_timeout = lock_timeout self._plugin_scheduler_task = plugin_scheduler_task self._tour_of_duty = PanoptesTourOfDuty(splay_percent=50) self._cycles_without_lock = 0
def __init__(self, resource, group_type, interval, creation_timestamp=None): assert PanoptesMetricValidators.valid_panoptes_resource( resource), u'resource must be an instance of PanoptesResource' assert PanoptesValidators.valid_nonempty_string( group_type), u'group_type must be a non-empty string' assert PanoptesValidators.valid_nonzero_integer( interval), u'interval must a integer greater than zero' self.__data = dict() self.__metrics_index = { metric_type: list() for metric_type in METRIC_TYPE_NAMES } self.__data[u'metrics_group_type'] = group_type self.__data[u'metrics_group_interval'] = interval self.__data[u'metrics_group_creation_timestamp'] = round(time(), METRICS_TIMESTAMP_PRECISION) \ if creation_timestamp is None else creation_timestamp self.__data[ u'metrics_group_schema_version'] = METRICS_GROUP_SCHEMA_VERSION self.__data[u'resource'] = resource self.__data[u'metrics'] = set() self.__data[u'dimensions'] = set() self._data_lock = threading.Lock()
def test_valid_nonzero_integer(self): self.assertFalse(PanoptesValidators.valid_nonzero_integer(-1)) self.assertFalse(PanoptesValidators.valid_nonzero_integer(0)) self.assertFalse(PanoptesValidators.valid_nonzero_integer(-0)) self.assertFalse(PanoptesValidators.valid_nonzero_integer(1.0)) self.assertFalse(PanoptesValidators.valid_nonzero_integer(u"")) self.assertFalse(PanoptesValidators.valid_nonzero_integer(False)) self.assertFalse(PanoptesValidators.valid_nonzero_integer(True)) self.assertTrue( PanoptesValidators.valid_nonzero_integer(2**(2**(2**(2**2)))))
def __init__(self, host, port, timeout, retries): assert PanoptesValidators.valid_nonempty_string(host), u'host must a non-empty string' assert PanoptesValidators.valid_port(port), u'port must be an integer between 1 and 65535' assert PanoptesValidators.valid_nonzero_integer(timeout), u'timeout must be a integer greater than zero' assert PanoptesValidators.valid_positive_integer(retries), u'retries must a non-negative integer' self._host = host self._port = port self._timeout = timeout self._retries = retries self._easy_snmp_session = None
def __init__(self, count=10, timeout=10, hostname='localhost'): assert PanoptesValidators.valid_nonzero_integer(count), 'count must be integer > 0' assert PanoptesValidators.valid_nonempty_string(hostname), 'hostname must be nonempty string' assert PanoptesValidators.valid_nonzero_integer(timeout), 'timeout must be integer > 0' super(PanoptesPingDirect, self).__init__() try: resp = subprocess.check_output( ['/bin/ping', '-c', str(count), '-w', str(timeout), hostname], stderr=subprocess.STDOUT, universal_newlines=True # return string not bytes ) self._get_ping_stats(resp) except subprocess.CalledProcessError as e: self._get_ping_stats(e.output) if self._response['packets_transmitted'] is not None: raise PanoptesPingTimeoutException("Ping timed out with response: " + str(self._response)) raise PanoptesPingException(e.output) except Exception as e: raise PanoptesPingException(str(e))
def __init__(self, tasks=1000, seconds=14400, memory_growth_mb=200, splay_percent=0): assert PanoptesValidators.valid_nonzero_integer( tasks), u'tasks must a integer greater than zero' assert PanoptesValidators.valid_nonzero_integer( seconds), u'seconds must a integer greater than zero' assert PanoptesValidators.valid_nonzero_integer( memory_growth_mb ), u'memory_growth_mb must a integer greater than zero' assert PanoptesValidators.valid_number( splay_percent), u'splay_percent must a number' assert 0 <= splay_percent <= 100, u'splay_percent must be a number between 0 and 100, inclusive' self._start_time = time.time() if platform.system() == u'Linux': # On Linux platforms, memory is reported in kilobytes self._memory_divider = 1024 else: # On all other platforms, assume bytes (true for Mac OS X) self._memory_divider = 1048576 self._initial_memory_mb = self._get_memory_utilization_in_mb() self._task_count = 0 self._tasks = tasks self._seconds = seconds self._memory_growth_mb = memory_growth_mb self._splay_percent = float(splay_percent) / 100 self._adjusted_tasks = round( self._tasks * (1 + (random.random() * self._splay_percent))) self._adjusted_seconds = round( self._seconds * (1 + (random.random() * self._splay_percent))) self._adjusted_memory_growth_mb = round( self._memory_growth_mb * (1 + (random.random() * self._splay_percent)))
def __init__(self, count=10, timeout=10, hostname=u'localhost'): assert PanoptesValidators.valid_nonzero_integer( count), u'count must be integer > 0' assert PanoptesValidators.valid_nonempty_string( hostname), u'hostname must be nonempty string' assert PanoptesValidators.valid_nonzero_integer( timeout), u'timeout must be integer > 0' self._response = dict() # dictionary containing ping statistics self._response[u'packets_transmitted'] = None self._response[u'packets_received'] = None self._response[u'packet_loss_pct'] = None self._response[u'execution_time'] = None self._response[u'round_trip_min'] = None self._response[u'round_trip_avg'] = None self._response[u'round_trip_max'] = None self._response[u'round_trip_stddev'] = None try: resp = subprocess.check_output( [ u'/bin/ping', u'-c', str(count), u'-w', str(timeout), hostname ], stderr=subprocess.STDOUT, universal_newlines=True # return string not bytes ) self._get_ping_stats(resp) except subprocess.CalledProcessError as e: self._get_ping_stats(e.output) if self._response[u'packets_transmitted'] is not None: raise PanoptesPingTimeoutException( u"Ping timed out with response: " + str(self._response)) raise PanoptesPingException(e.output) except Exception as e: raise PanoptesPingException(e.message)
def __init__(self, context, path, timeout, retries=1, identifier=None): """ Creates and maintains state for a lock Args: path (str): A '/' separated path for the lock timeout (int): in seconds. Must be a positive integer retries (int): how many times to try before giving up. Zero implies try forever identifier (str): Name to use for this lock contender. This can be useful for querying \ to see who the current lock contenders are Returns: PanoptesLock: lock """ assert PanoptesContextValidators.valid_panoptes_context( context), u'context must be a valid PanoptesContext' assert PanoptesValidators.valid_nonempty_string(path) and re.search(r"^/\S+", path), \ u'path must be a non-empty string that begins with /' assert PanoptesValidators.valid_nonzero_integer( timeout), u'timeout must be a positive integer' assert PanoptesValidators.valid_positive_integer( retries), u'retries must be a non-negative integer' assert PanoptesValidators.valid_nonempty_string( identifier), u'identifier must be a non-empty string' self._context = context self._logger = self._context.logger self._path = path self._timeout = timeout self._retries = retries self._identifier = identifier self._lock = None self._locked = False self._calling_module = get_calling_module_name(3) self._get_lock()
def get_lock(self, path, timeout, retries=1, identifier=None, listener=None): """ A wrapper around the kazoo library lock Args: path (str): A '/' separated path for the lock timeout (int): in seconds. Must be a positive integer retries (int): how many times to try before giving up. Zero implies try forever identifier (str): Name to use for this lock contender. This can be useful for querying \ to see who the current lock contenders are listener (callable): The callable to use to handle Zookeeper state changes Returns: kazoo.recipe.lock.Lock: lock """ assert PanoptesValidators.valid_nonempty_string(path) and re.search("^/\S+", path), \ u'path must be a non-empty string that begins with /' assert PanoptesValidators.valid_nonzero_integer( timeout), u'timeout must be a positive integer' assert PanoptesValidators.valid_positive_integer( retries), u'retries must be a non-negative integer' assert PanoptesValidators.valid_nonempty_string( identifier), u'identifier must be a non-empty string' assert ( not listener) or callable(listener), u'listener must be a callable' logger = self.logger calling_module = get_calling_module_name(2) logger.info(u"Creating lock for module: " + calling_module + u" with lock parameters: " u"path=" + path + u",timeout=" + str(timeout) + u",retries=" + str(retries) + u"," u"identifier=" + identifier) try: lock = self.zookeeper_client.Lock(path, identifier) except Exception as e: logger.error(u'Failed to create lock object: %s' % str(e)) return None if retries == 0: while True: logger.info( u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. ' % (identifier, path, lock.contenders())) try: lock.acquire(timeout=timeout) except LockTimeout: logger.info( u'Timed out after %d seconds trying to acquire lock. Retrying.' % timeout) except Exception as e: logger.info(u'Error in acquiring lock: %s. Retrying.' % str(e)) if lock.is_acquired: break else: tries = 0 while tries < retries: logger.info( u'Trying to acquire lock with client id "%s" under path %s. Other contenders: %s. ' % (identifier, path, lock.contenders())) try: lock.acquire(timeout=timeout) except LockTimeout: logger.info( u'Timed out after %d seconds trying to acquire lock. Retrying %d more times' % (timeout, retries - tries)) except Exception as e: logger.info( u'Error in acquiring lock: %s. Retrying %d more times' % (str(e), (retries - tries))) if lock.is_acquired: break tries += 1 if not lock.is_acquired: logger.warn(u'Unable to acquire lock after %d tries' % retries) if lock.is_acquired: logger.info(u'Lock acquired. Other contenders: %s' % lock.contenders()) if listener: self.zookeeper_client.add_listener(listener) return lock
def __init__(self, panoptes_context, consumer_type, topics, client_id, group, keys, poll_timeout, callback, validate=False, session_timeout=60, max_poll_records=500, max_partition_fetch_bytes=1048576): """ This class implements a helper/wrapper for writing Panoptes Consumers The consumer object produced by this class joins the group provided in the arguments, subscribes to the relevant topics (which are combination of the site name and consumer type), filters records by the provided keys, checks validity of the records by converting them to JSON and validating against consumer type specific schemas and then calls the callback function provided for each record. The consumer object also takes care of advancing the committed marker for each topic and partition - if the callback fails or returns false, the marker is not advanced. Args: panoptes_context (PanoptesContext): The PanoptesContext to be used by the consumer consumer_type (int): A valid consumer type topics (list): A valid consumer type client_id (str): A non-empty string that uniquely identifies the consumer instance - this is used for \ logging by and group administration by Kafka group (str): The Kafka group this consumer should be a part of keys (list, None): A comma separated list of keys the consumer should filter against poll_timeout (int): A non-negative integer which is the interval (in seconds) the consumer should sleep if \ no records are available on the Kafka bus callback (callable): A callable to which each processed, validated consumer records (deserialized JSON \ object) should be passed. The callable should return false if it cannot process the record due to temporary \ issues - it would be redelivered to the callback in that case validate (bool): Whether each incoming record should be validated. Defaults to False session_timeout (int): A non-negative integer which is the interval (in seconds) after which the Kafka \ Group Management system should consider the client disconnected max_poll_records (int): The maximum number of records to fetch in one poll cycle Returns: None """ assert isinstance( panoptes_context, PanoptesContext ), u'panoptes_context must be an instance of PanoptesContext' assert consumer_type in CONSUMER_TYPE_NAMES, u'consumer_type must be an valid attribute from ' \ u'PanoptesConsumerTypes' assert PanoptesValidators.valid_nonempty_iterable_of_strings( topics), u'' assert PanoptesValidators.valid_nonempty_string( client_id), u'client_id must be a non-empty string' assert keys is None or PanoptesValidators.valid_nonempty_iterable_of_strings(keys), \ u'keys must be None or a list of non-empty strings' assert PanoptesValidators.valid_positive_integer( poll_timeout), u'poll_timeout must be an integer' assert hasattr(callback, u'__call__'), u'callback must be a callable' assert isinstance(validate, bool), u'validate must be a boolean' assert PanoptesValidators.valid_nonzero_integer(session_timeout), u'session_timeout must be an integer '\ u'greater than zero' assert PanoptesValidators.valid_nonzero_integer(max_poll_records), u'max_poll_records must be an integer '\ u'greater than zero' self._panoptes_context = panoptes_context self._consumer_type = consumer_type self._topics = topics self._client_id = client_id self._keys = keys self._group = group self._poll_timeout = poll_timeout * 1000 self._session_timeout = session_timeout * 1000 self._request_timeout = self._session_timeout * 3 self._heartbeat_interval = old_div(self._session_timeout, 3) self._max_poll_records = max_poll_records self._max_partition_fetch_bytes = max_partition_fetch_bytes self._callback = callback self._validate = validate self._consumer = None self._last_polled = 0 self._asked_to_stop = False