def __init__(self, parent_pid):
"""
Initialize Luminosity
Create the :obj:`redis_conn` a Redis client object
Create the :obj:`self.anomalous_metrics` list
Create the :obj:`correlations` list
Create the :obj:`mysql_conn` MySQLConnection object
Create the :obj:`memcache_client` a constructor that does not make a
connection to memcached. The first call to a method on the object will
do that.
"""
super(Luminosity, self).__init__()
# @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
if settings.REDIS_PASSWORD:
self.redis_conn = StrictRedis(
password=settings.REDIS_PASSWORD,
unix_socket_path=settings.REDIS_SOCKET_PATH)
else:
self.redis_conn = StrictRedis(
unix_socket_path=settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.anomalous_metrics = Manager().list()
self.correlations = Manager().list()
self.mysql_conn = mysql.connector.connect(**config)
if settings.MEMCACHE_ENABLED:
self.memcache_client = pymemcache_Client(
(settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT),
connect_timeout=0.1,
timeout=0.2)
else:
self.memcache_client = None
def get_memcache_fp_ids_object(current_skyline_app, base_name):
"""
Return the fp_ids list from memcache if it exists.
.. warning: this returns a list and to mimic the MySQL results rows, when
this is used the list item must to turned into a dict to match the
MySQL/SQLAlchemy format.
"""
try:
pymemcache_Client
except:
from pymemcache.client.base import Client as pymemcache_Client
try:
literal_eval
except:
from ast import literal_eval
current_skyline_app_logger = str(current_skyline_app) + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
if settings.MEMCACHE_ENABLED:
memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2)
else:
return False
fp_ids_list_object_key = 'fp_ids_list_object.%s' % (str(base_name))
memcache_result = None
if settings.MEMCACHE_ENABLED:
try:
memcache_result = memcache_client.get(fp_ids_list_object_key)
except:
current_logger.error('error :: failed to get %s from memcache' % fp_ids_list_object_key)
try:
memcache_client.close()
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
except: # nosec
pass
result = None
if memcache_result:
try:
result = literal_eval(memcache_result)
except:
current_logger.error('error :: failed to process data from memcache key %s' % fp_ids_list_object_key)
result = False
try:
memcache_client.close()
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
except: # nosec
pass
return result
def __init__(self, parent_pid):
"""
Initialize Luminosity
Create the :obj:`redis_conn` a Redis client object
Create the :obj:`correlations` list
Create the :obj:`mysql_conn` MySQLConnection object
Create the :obj:`memcache_client` a constructor that does not make a
connection to memcached. The first call to a method on the object will
do that.
"""
super(Luminosity, self).__init__()
# @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
# @modified 20191030 - Bug #3266: py3 Redis binary objects not strings
# Branch #3262: py3
# Use get_redis_conn and get_redis_conn_decoded to use on Redis sets when the bytes
# types need to be decoded as utf-8 to str
# if settings.REDIS_PASSWORD:
# self.redis_conn = StrictRedis(password=settings.REDIS_PASSWORD, unix_socket_path=settings.REDIS_SOCKET_PATH)
# else:
# self.redis_conn = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
# @added 20191030 - Bug #3266: py3 Redis binary objects not strings
# Branch #3262: py3
# Added a single functions to deal with Redis connection and the
# charset='utf-8', decode_responses=True arguments required in py3
self.redis_conn = get_redis_conn(skyline_app)
self.redis_conn_decoded = get_redis_conn_decoded(skyline_app)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
# @modified 20190522 - Task #3034: Reduce multiprocessing Manager list usage
# Task #3032: Debug number of Python processes and memory use
# Branch #3002: docker
# Reduce amount of Manager instances that are used as each requires a
# copy of entire memory to be copied into each subprocess so this
# results in a python process per Manager instance, using as much
# memory as the parent. OK on a server, not so much in a container.
# Disabled all the Manager().list() below and replaced with Redis sets
# self.correlations = Manager().list()
# @added 20180720 - Task #2462: Implement useful metrics for Luminosity
# self.metrics_checked_for_correlation = Manager().list()
# self.runtimes = Manager().list()
self.mysql_conn = mysql.connector.connect(**config)
if settings.MEMCACHE_ENABLED:
self.memcache_client = pymemcache_Client(
(settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT),
connect_timeout=0.1,
timeout=0.2)
else:
self.memcache_client = None
this_host = str(os.uname()[1])
# Converting one settings variable into a local variable, just because it is a
# long string otherwise.
try:
ENABLE_IONOSPHERE_DEBUG = settings.ENABLE_IONOSPHERE_DEBUG
except:
logger.error(
'error :: layers :: cannot determine ENABLE_IONOSPHERE_DEBUG from settings'
% skyline_app)
ENABLE_IONOSPHERE_DEBUG = False
if settings.MEMCACHE_ENABLED:
memcache_client = pymemcache_Client(
(settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT),
connect_timeout=0.1,
timeout=0.2)
else:
memcache_client = None
LOCAL_DEBUG = False
def get_metrics_db_object(base_name):
"""
Returns the data of a metric from metrics table as an object and populates
memcached with a dict of the object
:param base_name: the metric base_name
:type base_name: str
:return: metrics_db_object
def get_memcache_metric_object(current_skyline_app, base_name):
"""
Return the metrics_db_object from memcache if it exists.
"""
try:
pymemcache_Client
except:
from pymemcache.client.base import Client as pymemcache_Client
try:
literal_eval
except:
from ast import literal_eval
current_skyline_app_logger = str(current_skyline_app) + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
if settings.MEMCACHE_ENABLED:
memcache_client = pymemcache_Client((settings.MEMCACHED_SERVER_IP, settings.MEMCACHED_SERVER_PORT), connect_timeout=0.1, timeout=0.2)
else:
return False
memcache_metrics_db_object = None
metrics_db_object_key = 'metrics_db_object.%s' % str(base_name)
memcache_result = None
if settings.MEMCACHE_ENABLED:
try:
memcache_result = memcache_client.get(metrics_db_object_key)
except:
current_logger.error('error :: failed to get %s from memcache' % metrics_db_object_key)
try:
memcache_client.close()
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
except: # nosec
pass
memcache_metric_dict = None
if memcache_result:
try:
memcache_metrics_db_object = literal_eval(memcache_result)
memcache_metric_dict = {}
# for k, v in memcache_result_list:
for k, v in memcache_metrics_db_object.iteritems():
key_name = str(k)
key_value = str(v)
memcache_metric_dict[key_name] = key_value
except:
current_logger.error('error :: failed to process data from memcache key %s' % metrics_db_object_key)
memcache_metric_dict = False
try:
memcache_client.close()
# @modified 20170913 - Task #2160: Test skyline with bandit
# Added nosec to exclude from bandit tests
except: # nosec
pass
if memcache_metric_dict:
metrics_id = int(memcache_metric_dict['id'])
metric_ionosphere_enabled = int(memcache_metric_dict['ionosphere_enabled'])
metrics_db_object = memcache_metric_dict
current_logger.info('get_memcache_metric_object :: returned memcache data for key - %s' % metrics_db_object_key)
return metrics_db_object
return False
