def __init__(self, parent_pid): """ Initialize Luminosity Create the :obj:`redis_conn` a Redis client object Create the :obj:`self.anomalous_metrics` list Create the :obj:`correlations` list Create the :obj:`mysql_conn` MySQLConnection object Create the :obj:`memcache_client` a constructor that does not make a connection to memcached. The first call to a method on the object will do that. """ super(Luminosity, self).__init__() # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow if settings.REDIS_PASSWORD: self.redis_conn = StrictRedis( password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH) else: self.redis_conn = StrictRedis( unix_socket_path=settings.REDIS_SOCKET_PATH) self.daemon = True self.parent_pid = parent_pid self.current_pid = getpid() self.anomalous_metrics = Manager().list() self.correlations = Manager().list() self.mysql_conn = mysql.connector.connect(**config) if settings.MEMCACHE_ENABLED: self.memcache_client = pymemcache_Client( (settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2) else: self.memcache_client = None
def get_memcache_fp_ids_object(current_skyline_app, base_name): """ Return the fp_ids list from memcache if it exists. .. warning: this returns a list and to mimic the MySQL results rows, when this is used the list item must to turned into a dict to match the MySQL/SQLAlchemy format. """ try: pymemcache_Client except: from pymemcache.client.base import Client as pymemcache_Client try: literal_eval except: from ast import literal_eval current_skyline_app_logger = str(current_skyline_app) + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) if settings.MEMCACHE_ENABLED: memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2) else: return False fp_ids_list_object_key = 'fp_ids_list_object.%s' % (str(base_name)) memcache_result = None if settings.MEMCACHE_ENABLED: try: memcache_result = memcache_client.get(fp_ids_list_object_key) except: current_logger.error('error :: failed to get %s from memcache' % fp_ids_list_object_key) try: memcache_client.close() # @modified 20170913 - Task #2160: Test skyline with bandit # Added nosec to exclude from bandit tests except: # nosec pass result = None if memcache_result: try: result = literal_eval(memcache_result) except: current_logger.error('error :: failed to process data from memcache key %s' % fp_ids_list_object_key) result = False try: memcache_client.close() # @modified 20170913 - Task #2160: Test skyline with bandit # Added nosec to exclude from bandit tests except: # nosec pass return result
def __init__(self, parent_pid): """ Initialize Luminosity Create the :obj:`redis_conn` a Redis client object Create the :obj:`correlations` list Create the :obj:`mysql_conn` MySQLConnection object Create the :obj:`memcache_client` a constructor that does not make a connection to memcached. The first call to a method on the object will do that. """ super(Luminosity, self).__init__() # @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow # @modified 20191030 - Bug #3266: py3 Redis binary objects not strings # Branch #3262: py3 # Use get_redis_conn and get_redis_conn_decoded to use on Redis sets when the bytes # types need to be decoded as utf-8 to str # if settings.REDIS_PASSWORD: # self.redis_conn = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH) # else: # self.redis_conn = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH) # @added 20191030 - Bug #3266: py3 Redis binary objects not strings # Branch #3262: py3 # Added a single functions to deal with Redis connection and the # charset='utf-8', decode_responses=True arguments required in py3 self.redis_conn = get_redis_conn(skyline_app) self.redis_conn_decoded = get_redis_conn_decoded(skyline_app) self.daemon = True self.parent_pid = parent_pid self.current_pid = getpid() # @modified 20190522 - Task #3034: Reduce multiprocessing Manager list usage # Task #3032: Debug number of Python processes and memory use # Branch #3002: docker # Reduce amount of Manager instances that are used as each requires a # copy of entire memory to be copied into each subprocess so this # results in a python process per Manager instance, using as much # memory as the parent. OK on a server, not so much in a container. # Disabled all the Manager().list() below and replaced with Redis sets # self.correlations = Manager().list() # @added 20180720 - Task #2462: Implement useful metrics for Luminosity # self.metrics_checked_for_correlation = Manager().list() # self.runtimes = Manager().list() self.mysql_conn = mysql.connector.connect(**config) if settings.MEMCACHE_ENABLED: self.memcache_client = pymemcache_Client( (settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2) else: self.memcache_client = None
this_host = str(os.uname()[1]) # Converting one settings variable into a local variable, just because it is a # long string otherwise. try: ENABLE_IONOSPHERE_DEBUG = settings.ENABLE_IONOSPHERE_DEBUG except: logger.error( 'error :: layers :: cannot determine ENABLE_IONOSPHERE_DEBUG from settings' % skyline_app) ENABLE_IONOSPHERE_DEBUG = False if settings.MEMCACHE_ENABLED: memcache_client = pymemcache_Client( (settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2) else: memcache_client = None LOCAL_DEBUG = False def get_metrics_db_object(base_name): """ Returns the data of a metric from metrics table as an object and populates memcached with a dict of the object :param base_name: the metric base_name :type base_name: str :return: metrics_db_object
def get_memcache_metric_object(current_skyline_app, base_name): """ Return the metrics_db_object from memcache if it exists. """ try: pymemcache_Client except: from pymemcache.client.base import Client as pymemcache_Client try: literal_eval except: from ast import literal_eval current_skyline_app_logger = str(current_skyline_app) + 'Log' current_logger = logging.getLogger(current_skyline_app_logger) if settings.MEMCACHE_ENABLED: memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2) else: return False memcache_metrics_db_object = None metrics_db_object_key = 'metrics_db_object.%s' % str(base_name) memcache_result = None if settings.MEMCACHE_ENABLED: try: memcache_result = memcache_client.get(metrics_db_object_key) except: current_logger.error('error :: failed to get %s from memcache' % metrics_db_object_key) try: memcache_client.close() # @modified 20170913 - Task #2160: Test skyline with bandit # Added nosec to exclude from bandit tests except: # nosec pass memcache_metric_dict = None if memcache_result: try: memcache_metrics_db_object = literal_eval(memcache_result) memcache_metric_dict = {} # for k, v in memcache_result_list: for k, v in memcache_metrics_db_object.iteritems(): key_name = str(k) key_value = str(v) memcache_metric_dict[key_name] = key_value except: current_logger.error('error :: failed to process data from memcache key %s' % metrics_db_object_key) memcache_metric_dict = False try: memcache_client.close() # @modified 20170913 - Task #2160: Test skyline with bandit # Added nosec to exclude from bandit tests except: # nosec pass if memcache_metric_dict: metrics_id = int(memcache_metric_dict['id']) metric_ionosphere_enabled = int(memcache_metric_dict['ionosphere_enabled']) metrics_db_object = memcache_metric_dict current_logger.info('get_memcache_metric_object :: returned memcache data for key - %s' % metrics_db_object_key) return metrics_db_object return False