def _redis_inst_up(self, r_inst, redish):
if not self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = redish
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.UP)
def __init__(self, conf, instance='0'):
self._conf = conf
module = Module.CONTRAIL_SNMP_COLLECTOR
self._moduleid = ModuleNames[module]
node_type = Module2NodeType[module]
self._node_type_name = NodeTypeNames[node_type]
self._hostname = socket.gethostname()
self._instance_id = instance
sandesh_global.init_generator(self._moduleid, self._hostname,
self._node_type_name, self._instance_id,
self._conf.collectors(),
self._node_type_name,
self._conf.http_port(),
['contrail_snmp_collector.gen_py',
'opserver.sandesh'],
self._conf._disc)
sandesh_global.set_logging_params(
enable_local_log=self._conf.log_local(),
category=self._conf.log_category(),
level=self._conf.log_level(),
file=self._conf.log_file(),
enable_syslog=self._conf.use_syslog(),
syslog_facility=self._conf.syslog_facility())
ConnectionState.init(sandesh_global, self._hostname, self._moduleid,
self._instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus)
self.if_stat = {}
self._logger = sandesh_global.logger()
def _redis_inst_down(self, r_inst):
if self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = None
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.DOWN)
def _publish_to_ifmap(self, oper_body):
try:
not_published = True
retry_count = 0
resp_xml = None
while not_published:
sess_id = self._mapclient.get_session_id()
req_xml = PublishRequest(sess_id, oper_body)
resp_xml = self._mapclient.call('publish', req_xml)
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if err_codes:
if retry_count == 0:
log_str = 'Error publishing to ifmap, req: %s, resp: %s' \
%(req_xml, resp_xml)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
ConnectionState.update(
conn_type = ConnectionType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.INIT,
message = 'Session lost, renew it',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
retry_count = retry_count + 1
self._init_conn()
if self._ifmap_restarted():
msg = "IF-MAP servers restarted, re-populate it"
self.config_log(msg, level=SandeshLevel.SYS_ERR)
self.reset()
self._get_api_server().publish_ifmap_to_discovery(
'down', msg)
self._publish_config_root()
self._db_client_mgr.db_resync()
self._publish_to_ifmap_enqueue('publish_discovery', 1)
else: # successful publish
not_published = False
break
# end while not_published
if retry_count:
log_str = 'Success publishing to ifmap after %d tries' \
%(retry_count)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
return True, resp_xml
except Exception as e:
# Failed to publish the operation due to unknown error.
# Probably a connection issue with the ifmap server.
msg = "Failed to publish request %s: %s" % (oper_body, str(e))
return False, msg
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr):
self.module = Module.COMPUTE_NODE_MGR
self.module_id = ModuleNames[self.module]
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global
EventManager.__init__(self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global)
self.node_type = "contrail-vrouter"
self.table = "ObjectVRouter"
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8102, ['vrouter.loadbalancer',
'nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
self.supervisor_serverurl = "unix:///var/run/supervisord_vrouter.sock"
self.add_current_process()
ConnectionState.init(sandesh_global, socket.gethostname(), self.module_id,
self.instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
self.lb_stats = LoadbalancerStatsUVE()
self.send_system_cpu_info()
self.third_party_process_dict = {}
def __init__(self, conf):
self._conf = conf
module = Module.CONTRAIL_TOPOLOGY
self._moduleid = ModuleNames[module]
node_type = Module2NodeType[module]
self._node_type_name = NodeTypeNames[node_type]
self._hostname = socket.gethostname()
self._instance_id = '0'
if self._conf.sandesh_send_rate_limit() is not None:
SandeshSystem.set_sandesh_send_rate_limit( \
self._conf.sandesh_send_rate_limit())
sandesh_global.init_generator(self._moduleid, self._hostname,
self._node_type_name, self._instance_id,
self._conf.collectors(),
self._node_type_name,
self._conf.http_port(),
['contrail_topology.sandesh'],
self._conf._disc)
sandesh_global.set_logging_params(
enable_local_log=self._conf.log_local(),
category=self._conf.log_category(),
level=self._conf.log_level(),
file=self._conf.log_file(),
enable_syslog=self._conf.use_syslog(),
syslog_facility=self._conf.syslog_facility())
ConnectionState.init(sandesh_global, self._hostname, self._moduleid,
self._instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus)
def sandesh_init(self):
""" Init Sandesh """
self._sandesh = Sandesh()
# Register custom sandesh request handlers.
self._redefine_sandesh_handles()
# Initialize Sandesh generator.
self._sandesh.init_generator(
self._module["name"], self._module["hostname"],
self._module["node_type_name"], self._module["instance_id"],
self._args.random_collectors, 'kube_manager_context',
int(self._args.http_server_port),
['cfgm_common', 'kube_manager'],
logger_class=self._args.logger_class,
logger_config_file=self._args.logging_conf,
config=self._args.sandesh_config)
# Set Sandesh logging params.
self._sandesh.set_logging_params(
enable_local_log=self._args.log_local,
category=self._args.log_category, level=self._args.log_level,
file=self._args.log_file, enable_syslog=self._args.use_syslog,
syslog_facility=self._args.syslog_facility)
# Connect to collector.
ConnectionState.init(
self._sandesh, self._module["hostname"], self._module["name"],
self._module["instance_id"],
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self._module["table"])
def sandesh_init(self, http_server_port=None):
""" Init sandesh """
self._sandesh = Sandesh()
self.redefine_sandesh_handles()
if not http_server_port:
http_server_port = self._args.http_server_port
self._sandesh.init_generator(
self._module_name, self._hostname, self._node_type_name,
self._instance_id, self._args.random_collectors,
'%s_context' % self.context, int(http_server_port),
['cfgm_common', '%s.sandesh' % self.module_pkg],
logger_class=self._args.logger_class,
logger_config_file=self._args.logging_conf,
config=self._args.sandesh_config)
self._sandesh.set_logging_params(
enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level,
file=self._args.log_file,
enable_syslog=self._args.use_syslog,
syslog_facility=self._args.syslog_facility)
# connection state init
ConnectionState.init(
self._sandesh, self._hostname, self._module_name,
self._instance_id,
staticmethod(ConnectionState.get_conn_state_cb),
NodeStatusUVE, NodeStatus, self.table)
VncGreenlet.register_sandesh_handler()
def run(self):
lck = False
while True:
try:
k, value = self._redis.brpop("DELETED")
self._sem.acquire()
lck = True
self._logger.debug("%s del received for " % value)
# value is of the format:
# DEL:<key>:<src>:<node-type>:<module>:<instance-id>:<message-type>:<seqno>
self._redis.delete(value)
except redis.exceptions.ResponseError:
#send redis connection down msg. Coule be bcos of authentication
ConnectionState.update(conn_type = ConnectionType.REDIS,
name = 'UVE', status = ConnectionStatus.DOWN,
message = 'UVE result : Connection Error',
server_addrs = ['%s:%d' % (self._local_redis_uve[0],
self._local_redis_uve[1])])
sys.exit()
except redis.exceptions.ConnectionError:
if lck:
self._sem.release()
lck = False
gevent.sleep(5)
else:
if lck:
self._sem.release()
lck = False
self._logger.debug("Deleted %s" % value)
self._logger.debug("UVE %s Type %s" % (key, typ))
def conn_state_notify(self, svc, msg='', up=True, servers=''):
ctype = self.map_svc(svc)
status = self.map_sts(up)
if isinstance(servers, basestring):
servers = [servers]
ConnectionState.update(conn_type=ctype, name='SNMP', status=status,
message=msg, server_addrs=servers)
def heartbeat(self):
while True:
# send heartbeat for each published object seperately
# dictionary can change size during iteration
pub_list = self.pubdata.copy()
for cookie in pub_list:
payload = {'cookie': cookie}
self.syslog('Sending cookie %s in heartbeat' % cookie)
try:
r = requests.post(
self.hburl, data=json.dumps(payload), headers=self._headers)
except requests.exceptions.ConnectionError:
service, data = pub_list[cookie]
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = service, status = ConnectionState.DOWN,
server_addrs = ['%s:%s' % (self._server_ip, \
self._server_port)],
message = 'HeartBeat - Connection Error')
self.syslog('Connection Error')
continue
# if DS lost track of our data, republish
if r.status_code == 404:
# forget cached cookie and object; will re-learn
self.syslog('Server lost track of token %s' % (cookie))
service, data = self.pubdata[cookie]
del self.pubdata[cookie]
self._publish_int(service, data)
gevent.sleep(HC_INTERVAL)
def _cassandra_init_conn_pools(self):
for ks, cf_list in self._keyspaces.items():
pool = pycassa.ConnectionPool(
ks,
self._server_list,
max_overflow=-1,
use_threadlocal=True,
prefill=True,
pool_size=20,
pool_timeout=120,
max_retries=-1,
timeout=5,
)
rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
for (cf, _) in cf_list:
self._cf_dict[cf] = ColumnFamily(
pool, cf, read_consistency_level=rd_consistency, write_consistency_level=wr_consistency
)
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="Cassandra",
status=ConnectionStatus.UP,
message="",
server_addrs=self._server_list,
)
self._conn_state = ConnectionStatus.UP
msg = "Cassandra connection ESTABLISHED"
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def update_redis_uve_list(self, redis_uve_list):
newlist = set(redis_uve_list)
chg = False
# if some redis instances are gone, remove them from our map
for test_elem in self._redis_uve_map.keys():
if test_elem not in newlist:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
del self._redis_uve_map[test_elem]
ConnectionState.delete(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port))
# new redis instances need to be inserted into the map
for test_elem in newlist:
if test_elem not in self._redis_uve_map:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
self._redis_uve_map[test_elem] = None
ConnectionState.update(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port), ConnectionStatus.INIT)
if chg:
self._logger.error("updated redis_uve_list %s" % str(self._redis_uve_map))
# Exercise redis connections to update health
if len(newlist):
self.get_uve("ObjectCollectorInfo:__NONE__", False, None)
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr):
self.node_type = "contrail-config"
self.module = Module.CONFIG_NODE_MGR
self.module_id = ModuleNames[self.module]
self.supervisor_serverurl = "unix:///var/run/supervisord_config.sock"
self.add_current_process()
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global
EventManager.__init__(
self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global)
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8100, ['cfgm_common.uve'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
ConnectionState.init(sandesh_global, socket.gethostname(),
self.module_id, self.instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus)
self.send_system_cpu_info()
self.third_party_process_list = [ ]
def __init__(self, conf):
self._conf = conf
module = Module.CONTRAIL_TOPOLOGY
self._moduleid = ModuleNames[module]
node_type = Module2NodeType[module]
self._node_type_name = NodeTypeNames[node_type]
self._hostname = socket.gethostname()
self.table = "ObjectCollectorInfo"
self._instance_id = '0'
sandesh_global.init_generator(self._moduleid, self._hostname,
self._node_type_name, self._instance_id,
self._conf.random_collectors,
self._node_type_name,
self._conf.http_port(),
['contrail_topology.sandesh'],
config=self._conf.sandesh_config())
sandesh_global.set_logging_params(
enable_local_log=self._conf.log_local(),
category=self._conf.log_category(),
level=self._conf.log_level(),
file=self._conf.log_file(),
enable_syslog=self._conf.use_syslog(),
syslog_facility=self._conf.syslog_facility())
ConnectionState.init(sandesh_global, self._hostname, self._moduleid,
self._instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
self._logger = sandesh_global.logger()
def sandesh_init(self):
""" Init sandesh """
self._sandesh = Sandesh()
# Reset the sandesh send rate limit value
if self._args.sandesh_send_rate_limit is not None:
SandeshSystem.set_sandesh_send_rate_limit(
self._args.sandesh_send_rate_limit)
self.redefine_sandesh_handles()
self._sandesh.init_generator(
self._module_name, self._hostname, self._node_type_name,
self._instance_id, self._args.random_collectors,
'%s_context' % self.context, int(self._args.http_server_port),
['cfgm_common', '%s.sandesh' % self.module_pkg], self.discovery,
logger_class=self._args.logger_class,
logger_config_file=self._args.logging_conf)
self._sandesh.set_logging_params(
enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level,
file=self._args.log_file,
enable_syslog=self._args.use_syslog,
syslog_facility=self._args.syslog_facility)
# connection state init
ConnectionState.init(
self._sandesh, self._hostname, self._module_name,
self._instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
def __init__(self, db, discovery, args=None):
self._args = args
self._db = db
module = Module.SVC_MONITOR
node_type = Module2NodeType[module]
self._module_name = ModuleNames[module]
self._node_type_name = NodeTypeNames[node_type]
self._instance_id = INSTANCE_ID_DEFAULT
self._hostname = socket.gethostname()
#sandesh init
self._sandesh = self._sandesh_init(discovery)
# connection state init
ConnectionState.init(self._sandesh, self._hostname, self._module_name,
self._instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus)
#create cpu_info object to send periodic updates
sysinfo_req = False
cpu_info = vnc_cpu_info.CpuInfo(self._module_name,
self._instance_id, sysinfo_req, self._sandesh, 60)
self._cpu_info = cpu_info
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr):
EventManager.__init__(
self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global)
self.node_type = 'contrail-analytics'
self.table = "ObjectCollectorInfo"
self.module = Module.ANALYTICS_NODE_MGR
self.module_id = ModuleNames[self.module]
self.supervisor_serverurl = "unix:///var/run/supervisord_analytics.sock"
self.add_current_process()
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8104, ['analytics', 'nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
ConnectionState.init(sandesh_global, socket.gethostname(), self.module_id,
self.instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
self.send_system_cpu_info()
self.third_party_process_list = [ ]
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr):
self.node_type = "contrail-control"
self.uve_node_type = UVENodeTypeNames[NodeType.CONTROL]
self.table = "ObjectBgpRouter"
self.module = Module.CONTROL_NODE_MGR
self.module_id = ModuleNames[self.module]
self.supervisor_serverurl = "unix:///var/run/supervisord_control.sock"
self.add_current_process()
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global;
EventManager.__init__(
self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global)
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8101, ['nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
ConnectionState.init(sandesh_global, socket.gethostname(), self.module_id,
self.instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
self.send_init_info()
self.third_party_process_dict = {}
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnType.APISERVER, name='ApiServer',
status=status,
message=message or 'ApiServer Connection State updated',
server_addrs=['%s:%s' % (self._args.api_server_ip,
self._args.api_server_port)])
def sandesh_init(self):
""" Init Sandesh """
self._sandesh = Sandesh()
# Initialize Sandesh generator.
self._sandesh.init_generator(
self.module['name'], self.module['hostname'],
self.module['node_type_name'], self.module['instance_id'],
self._args.collectors, 'mesos_manager_context',
int(self._args.http_server_port),
['cfgm_common', 'mesos_manager.sandesh'],
logger_class=self._args.logger_class,
logger_config_file=self._args.logging_conf,
config=self._args.sandesh_config)
# Set Sandesh logging params.
self._sandesh.set_logging_params(
enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level, file=self._args.log_file,
enable_syslog=self._args.use_syslog,
syslog_facility=self._args.syslog_facility)
# Connect to collector.
ConnectionState.init(self._sandesh, self.module['hostname'],
self.module['name'], self.module['instance_id'],
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.module['table'])
def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh):
global openstack_driver
openstack_driver = self
if api_server_ip == '0.0.0.0':
self._vnc_api_ip = '127.0.0.1'
else:
self._vnc_api_ip = api_server_ip
self._vnc_api_port = api_server_port
self._config_sections = conf_sections
fill_keystone_opts(self, conf_sections)
if 'v3' in self._auth_url.split('/')[-1]:
self._ks_domains_list = self._ksv3_domains_list
self._ks_domain_get = self._ksv3_domain_get
self._ks_projects_list = self._ksv3_projects_list
self._ks_project_get = self._ksv3_project_get
self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc
self._add_project_to_vnc = self._ksv3_add_project_to_vnc
self._del_project_from_vnc = self._ksv3_del_project_from_vnc
self._vnc_default_domain_id = None
else:
self._ks_domains_list = None
self._ks_domain_get = None
self._ks_projects_list = self._ksv2_projects_list
self._ks_project_get = self._ksv2_project_get
self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc
self._add_project_to_vnc = self._ksv2_add_project_to_vnc
self._del_project_from_vnc = self._ksv2_del_project_from_vnc
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.INIT, message='',
server_addrs=[self._auth_url])
self._vnc_lib = None
# resync failures, don't retry forever
self._failed_domain_dels = set()
self._failed_project_dels = set()
# active domains/projects in contrail/vnc api server
self._vnc_domain_ids = set()
self._vnc_project_ids = set()
# logging
self._sandesh_logger = sandesh.logger()
self._vnc_os_logger = logging.getLogger(__name__)
self._vnc_os_logger.setLevel(logging.ERROR)
# Add the log message handler to the logger
try:
with open(self._err_file, 'a'):
handler = logging.handlers.RotatingFileHandler(
self._err_file, maxBytes=64*1024, backupCount=5)
self._vnc_os_logger.addHandler(handler)
except IOError:
self._sandesh_logger.error("Failed to open trace file %s" %
self._err_file)
self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
def _update_sandesh_status(self, status, msg=""):
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="RabbitMQ",
status=status,
message=msg,
server_addrs=["%s:%s" % (self._rabbit_ip, self._rabbit_port)],
)
def _update_sandesh_status(self, status, msg=""):
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="Cassandra",
status=status,
message=msg,
server_addrs=self._server_list,
)
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnectionType.APISERVER,
name="ApiServer",
status=status,
message=message or "",
server_addrs=["%s:%s" % (self._args.api_server_ip, self._args.api_server_port)],
)
def _sandesh_connection_info_update(self, status, message):
from pysandesh.connection_info import ConnectionState
from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \
ConnectionType
ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
name = 'Zookeeper', status = getattr(ConnectionStatus, status),
message = message,
server_addrs = self._server_list.split(','))
def _query(self):
conn_state_updated = False
# hoping all errors are transient and a little wait will solve the problem
while True:
try:
self.stats['request'] += 1
r = requests.post(
self.url, data=self.post_body, headers=self._headers, timeout=5)
if r.status_code == 200:
break
self.inc_stats('sc_%d' % r.status_code)
emsg = "Status Code %d" % r.status_code
except requests.exceptions.ConnectionError:
self.stats['conn_error'] += 1
emsg = 'Connection Error'
except (requests.exceptions.Timeout, socket.timeout):
self.stats['timeout'] += 1
emsg = 'Request Timeout'
self.syslog('connection error or failed to subscribe')
if not conn_state_updated:
conn_state_updated = True
ConnectionState.update(
conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.DOWN,
message = 'Subscribe - %s' % emsg,
server_addrs = \
['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
gevent.sleep(2)
# end while
self.syslog('query resp => %s ' % r.text)
response = r.json()
# avoid signature on ttl which can change between iterations
info = response[self.service_type]
infostr = json.dumps(info)
sig = hashlib.md5(infostr).hexdigest()
self.stats['response'] += 1
self.stats['ttl'] = response['ttl']
self.stats['blob'] = infostr
self.ttl = response['ttl']
self.change = False
if sig != self.sig:
#print 'signature mismatch! old=%s, new=%s' % (self.sig, sig)
self.info = info
self.sig = sig
self.change = True
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.UP,
message = 'Subscribe Response',
server_addrs = ['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnectionType.APISERVER,
name='ApiServer',
status=status,
message=message or '',
server_addrs=[
'%s:%s' %
(self._args.api_server_ip, self._args.api_server_port)
])
def conn_state_notify(self, svc, msg='', up=True, servers=''):
ctype = self.map_svc(svc)
status = self.map_sts(up)
if isinstance(servers, basestring):
servers = [servers]
ConnectionState.update(conn_type=ctype,
name='SNMP',
status=status,
message=msg,
server_addrs=servers)
def _get_keystone_conn(self):
if self._ks:
return
if self._use_certs:
verify=self._kscertbundle
else:
if self._insecure:
verify=False
else:
verify=True
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'project_name': self._admin_tenant,
'username': self._auth_user,
'password': self._auth_passwd,
'user_domain_name': self._user_domain_name,
}
if self._user_domain_name:
kwargs.update({
'project_domain_name': self._project_domain_name,
'project_name': self._project_name,
})
else:
kwargs.update({
'domain_id': self._domain_id,
})
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
try:
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
except kexceptions.DiscoveryFailure:
# Probably a v2 Keytone API, remove v3 args and try again
v3_args = ['user_domain_name', 'project_domain_name', 'domain_id']
for arg in v3_args:
kwargs.pop(arg, None)
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone',
status=ConnectionStatus.UP, message='',
server_addrs=[self._auth_url])
def connection_state_update(status, message=None):
ConnectionState.update(conn_type=ConnType.APISERVER,
name='ApiServer',
status=status,
message=message or '',
server_addrs=[
'%s:%s' %
(args.api_server_ip, args.api_server_port)
])
st_logger.notice(
"############# UPDATE CONNECTION STATE: {}".format(status))
def _ksv3_domain_get(self, id=None):
try:
return {'name': self._ks.domains.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.domains.get(id).name}
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr,
hostip, db_port, minimum_diskgb, contrail_databases,
cassandra_repair_interval,
cassandra_repair_logdir, **dss_kwargs):
self.node_type = "contrail-database"
self.uve_node_type = UVENodeTypeNames[NodeType.DATABASE]
self.table = "ObjectDatabaseInfo"
self.module = Module.DATABASE_NODE_MGR
self.module_id = ModuleNames[self.module]
self.hostip = hostip
self.db_port = db_port
self.minimum_diskgb = minimum_diskgb
self.contrail_databases = contrail_databases
self.cassandra_repair_interval = cassandra_repair_interval
self.cassandra_repair_logdir = cassandra_repair_logdir
self.cassandra_mgr = CassandraManager(self.cassandra_repair_logdir,
'analyticsDb', self.table,
self.contrail_databases,
self.hostip, self.minimum_diskgb,
self.db_port)
if os.path.exists('/tmp/supervisord_database.sock'):
self.supervisor_serverurl = "unix:///tmp/supervisord_database.sock"
else:
self.supervisor_serverurl = "unix:///var/run/supervisord_database.sock"
self.add_current_process()
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global
EventManager.__init__(
self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global, **dss_kwargs)
self.sandesh_global = sandesh_global
if self.rule_file is '':
self.rule_file = "/etc/contrail/" + \
"supervisord_database_files/contrail-database.rules"
json_file = open(self.rule_file)
self.rules_data = json.load(json_file)
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(), node_type_name,
self.instance_id, self.collector_addr, self.module_id, 8103,
['database.sandesh', 'nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
ConnectionState.init(sandesh_global, socket.gethostname(), self.module_id,
self.instance_id,
staticmethod(ConnectionState.get_conn_state_cb),
NodeStatusUVE, NodeStatus, self.table,
self.get_process_state_cb)
self.send_init_info()
self.third_party_process_dict = {}
self.third_party_process_dict["cassandra"] = "Dcassandra-pidfile=.*cassandra\.pid"
self.third_party_process_dict["zookeeper"] = "org.apache.zookeeper.server.quorum.QuorumPeerMain"
def _ksv3_domain_get(self, id=None):
try:
return {'name': self._ks.domains.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.domains.get(id).name}
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr,
hostip, db_port, minimum_diskgb, contrail_databases,
cassandra_repair_interval,
cassandra_repair_logdir,
**kwargs):
self.node_type = "contrail-config"
self.uve_node_type = UVENodeTypeNames[NodeType.CONFIG]
self.table = "ObjectConfigNode"
self.module = Module.CONFIG_NODE_MGR
self.module_id = ModuleNames[self.module]
self.cassandra_repair_interval = cassandra_repair_interval
self.cassandra_repair_logdir = cassandra_repair_logdir
self.hostip = hostip
self.db_port = db_port
self.minimum_diskgb = minimum_diskgb
self.contrail_databases = contrail_databases
self.cassandra_mgr = CassandraManager(self.cassandra_repair_logdir,
'configDb', self.table,
self.contrail_databases,
self.hostip, self.minimum_diskgb,
self.db_port)
self.cassandra_mgr.use_ssl = kwargs.get('cassandra_use_ssl', False)
if os.path.exists('/tmp/supervisord_config.sock'):
self.supervisor_serverurl = "unix:///tmp/supervisord_config.sock"
else:
self.supervisor_serverurl = "unix:///var/run/supervisord_config.sock"
self.db = package_installed('contrail-openstack-database')
self.config_db = package_installed('contrail-database-common')
self.add_current_process()
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global
EventManager.__init__(
self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global,
**kwargs)
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8100, ['database.sandesh', 'nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
ConnectionState.init(sandesh_global, socket.gethostname(),
self.module_id, self.instance_id,
staticmethod(ConnectionState.get_conn_state_cb),
NodeStatusUVE, NodeStatus, self.table,
self.get_process_state_cb)
self.send_init_info()
self.third_party_process_dict = {}
self.third_party_process_dict["cassandra"] = "Dcassandra-pidfile=.*cassandra\.pid"
self.third_party_process_dict["zookeeper"] = "org.apache.zookeeper.server.quorum.QuorumPeerMain"
def __init__(self, dc, service_type, count, f=None, *args, **kw):
self.dc = dc
self.f = f
self.kw = kw
self.args = args
self.count = count
self.service_type = service_type
self._headers = {
'Content-type': 'application/json',
}
self.info = []
infostr = json.dumps(self.info)
self.sig = hashlib.md5(infostr).hexdigest()
self.done = False
self.stats = {
'service_type': service_type,
'request': 0,
'response': 0,
'conn_error': 0,
'timeout': 0,
'exc_unknown': 0,
'exc_info': '',
'instances': count,
'ttl': 0,
'blob': '',
}
data = {
'service': service_type,
'instances': count,
'client-type': dc._client_type,
'remote-addr': dc._remote_addr,
'client': dc._myid
}
self.post_body = json.dumps(data)
self.url = "http://%s:%s/subscribe" % (dc._server_ip, dc._server_port)
if f:
# asynch - callback when new info is received
ConnectionState.update(
conn_type=ConnectionType.DISCOVERY,
name=self.service_type,
status=ConnectionStatus.INIT,
message='Subscribe',
server_addrs=['%s:%s' % (dc._server_ip, dc._server_port)])
self.task = gevent.spawn(self.ttl_loop)
else:
self._query()
self.done = True
def _init_conn(self):
self._mapclient = client(
("%s" % (self._ifmap_srv_ip), "%s" % (self._ifmap_srv_port)),
self._username, self._password, self._NAMESPACES,
self._ssl_options)
connected = False
while not connected:
try:
resp_xml = self._mapclient.call('newSession',
NewSessionRequest())
except socket.error as e:
msg = 'Failed to establish IF-MAP connection: %s' % str(e)
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
continue
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if not err_codes:
connected = True
else:
msg = "Failed to establish IF-MAP connection: %s" % err_codes
self.config_log(msg, level=SandeshLevel.SYS_WARN)
session_id = self._mapclient.get_session_id()
try:
self._mapclient.call('endSession',
EndSessionRequest(session_id))
except socket.error as e:
msg = "Failed to end the IF-MAP session %s: %s" %\
(session_id, str(e))
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
ConnectionState.update(conn_type=ConnectionType.IFMAP,
name='IfMap',
status=ConnectionStatus.UP,
message='',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
self._conn_state = ConnectionStatus.UP
msg = 'IFMAP connection ESTABLISHED'
self.config_log(msg, level=SandeshLevel.SYS_NOTICE)
self._mapclient.set_session_id(
newSessionResult(resp_xml).get_session_id())
self._mapclient.set_publisher_id(
newSessionResult(resp_xml).get_publisher_id())
def __init__(self, dc, service_type, count, f=None, *args, **kw):
self.dc = dc
self.f = f
self.kw = kw
self.args = args
self.count = count
self.service_type = service_type
self._headers = {
'Content-type': 'application/json',
}
self.info = []
infostr = json.dumps(self.info)
self.sig = hashlib.md5(infostr).hexdigest()
self.done = False
self.stats = {
'service_type' : service_type,
'request' : 0,
'response' : 0,
'conn_error' : 0,
'timeout' : 0,
'exc_unknown' : 0,
'exc_info' : '',
'instances' : count,
'ttl' : 0,
'blob' : '',
}
data = {
'service': service_type,
'instances': count,
'client-type': dc._client_type,
'remote-addr': dc._myip,
'client': dc._myid
}
self.post_body = json.dumps(data)
self.url = "http://%s:%s/subscribe" % (dc._server_ip, dc._server_port)
if f:
# asynch - callback when new info is received
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.INIT,
message = 'Subscribe',
server_addrs = ['%s:%s' % (dc._server_ip, dc._server_port)])
self.task = gevent.spawn(self.ttl_loop)
else:
self._query()
self.done = True
def _resync_all_domains(self):
if not self._ks_domains_list:
# < keystonev3, no domains
return False
self._get_keystone_conn()
# compare new and old set,
# optimize for common case where nothing has changed,
# so track the project-ids in a set add '-',
# keystone gives uuid without...
try:
# The Default domain in ks(for v2 support) has id of 'default'
# replace with uuid of default-domain in vnc
ks_domain_ids = set([
str(uuid.UUID(dom['id'])) for dom in self._ks_domains_list()
if dom['id'] != 'default'
])
ks_domain_ids.add(self._vnc_default_domain_id)
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %
(e, datetime.utcnow()),
server_addrs=[self._auth_url])
return True # retry
vnc_domain_ids = self._vnc_domain_ids
if vnc_domain_ids == ks_domain_ids:
# no change, go back to poll
return False
for vnc_domain_id in vnc_domain_ids - ks_domain_ids:
self.q.put((Q_DELETE, 'domain', vnc_domain_id))
if self._keystone_sync_on_demand:
# pre_domain_read will get it
pass
else:
for ks_domain_id in ks_domain_ids - vnc_domain_ids:
self.q.put((Q_CREATE, 'domain', ks_domain_id))
self.q.join()
gevent.sleep(0)
# we are in sync
self._vnc_domain_ids = ks_domain_ids
return False
def _get_keystone_conn(self):
if self._ks:
return
verify = self._kscertbundle if self._use_certs else not self._insecure
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'username': self._auth_user,
'password': self._auth_passwd,
}
# Add user domain info
kwargs.update(
**cfgmutils.get_user_domain_kwargs(self._config_sections))
# Get project scope auth params
scope_kwargs = cfgmutils.get_project_scope_kwargs(
self._config_sections)
if not scope_kwargs:
# Default to domain scoped auth
scope_kwargs = cfgmutils.get_domain_scope_kwargs(
self._config_sections)
kwargs.update(**scope_kwargs)
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
try:
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
except kexceptions.DiscoveryFailure:
# Probably a v2 Keytone API, remove v3 args and try again
v3_args = ['user_domain_name', 'project_domain_name', 'domain_id']
for arg in v3_args:
kwargs.pop(arg, None)
kwargs['project_name'] = self._admin_tenant
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.UP,
message='',
server_addrs=[self._auth_url])
def _get_keystone_conn(self):
if self._ks:
return
verify = self._kscertbundle if self._use_certs else self._insecure
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'project_name': self._admin_tenant,
'username': self._auth_user,
'password': self._auth_passwd,
'user_domain_name': self._user_domain_name,
}
if self._user_domain_name:
kwargs.update({
'project_domain_name': self._project_domain_name,
'project_name': self._project_name,
})
else:
kwargs.update({
'domain_id': self._domain_id,
})
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
try:
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
except kexceptions.DiscoveryFailure:
# Probably a v2 Keytone API, remove v3 args and try again
v3_args = ['user_domain_name', 'project_domain_name', 'domain_id']
for arg in v3_args:
kwargs.pop(arg, None)
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.UP,
message='',
server_addrs=[self._auth_url])
def run(self):
exitrun = False
while not exitrun:
for rkey, rinst in self._redis_uve_map.iteritems():
old_pid = rinst.collector_pid
try:
if rinst.redis_handle is None:
rinst.redis_handle = redis.StrictRedis(
host=rkey.ip,
port=rkey.port,
password=self._redis_password,
db=1,
socket_timeout=90)
rinst.collector_pid = None
# check for known collector pid string
# if there's a mismatch, we must read it again
if rinst.collector_pid is not None:
if not rinst.redis_handle.sismember(
"NGENERATORS", rinst.collector_pid):
rinst.collector_pid = None
# read the collector pid string
if rinst.collector_pid is None:
for gen in rinst.redis_handle.smembers("NGENERATORS"):
module = gen.split(':')[2]
if module == "contrail-collector":
rinst.collector_pid = gen
except gevent.GreenletExit:
self._logger.error('UVEServer Exiting on gevent-kill')
exitrun = True
break
except Exception as e:
self._logger.error("redis/collector healthcheck failed %s for %s" \
% (str(e), str(rkey)))
rinst.redis_handle = None
rinst.collector_pid = None
finally:
# Update redis/collector health
if old_pid is None and rinst.collector_pid is not None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.UP,
[rkey.ip+":"+str(rkey.port)])
if old_pid is not None and rinst.collector_pid is None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.DOWN,
[rkey.ip+":"+str(rkey.port)])
if not exitrun:
gevent.sleep(self._freq)
def run_introspect(cfg, database, lock):
sandesh_config = cfg['sandesh']
sandesh_config['collectors'] = sandesh_config['collectors'].split()
random.shuffle(sandesh_config['collectors'])
sandesh_config.update({
'id': Module.VCENTER_MANAGER,
'hostname': socket.gethostname(),
'table': 'ObjectContrailvCenterManagerNode',
'instance_id': INSTANCE_ID_DEFAULT,
'introspect_port': ServiceHttpPortMap['contrail-vcenter-manager'],
})
sandesh_config['name'] = ModuleNames[sandesh_config['id']]
sandesh_config['node_type'] = Module2NodeType[sandesh_config['id']]
sandesh_config['node_type_name'] = NodeTypeNames[sandesh_config['node_type']]
sandesh = Sandesh()
sandesh_handler = SandeshHandler(database, lock)
sandesh_handler.bind_handlers()
config = SandeshConfig(http_server_ip=sandesh_config['http_server_ip'])
sandesh.init_generator(
module='cvm',
source=sandesh_config['hostname'],
node_type=sandesh_config['node_type_name'],
instance_id=sandesh_config['instance_id'],
collectors=sandesh_config['collectors'],
client_context='cvm_context',
http_port=sandesh_config['introspect_port'],
sandesh_req_uve_pkg_list=['cfgm_common', 'cvm'],
config=config
)
sandesh.sandesh_logger().set_logger_params(
logger=sandesh.logger(),
enable_local_log=True,
level=translate_logging_level(sandesh_config['logging_level']),
file=sandesh_config['log_file'],
enable_syslog=False,
syslog_facility=None
)
ConnectionState.init(
sandesh=sandesh,
hostname=sandesh_config['hostname'],
module_id=sandesh_config['name'],
instance_id=sandesh_config['instance_id'],
conn_status_cb=staticmethod(ConnectionState.get_conn_state_cb),
uve_type_cls=NodeStatusUVE,
uve_data_type_cls=NodeStatus,
table=sandesh_config['table']
)
def send_nodemgr_process_status(self):
if self.prev_fail_status_bits == self.fail_status_bits:
return
self.prev_fail_status_bits = self.fail_status_bits
fail_status_bits = self.fail_status_bits
state, description = self._get_process_state(fail_status_bits)
conn_infos = ConnectionState._connection_map.values()
(cb_state, cb_description) = ConnectionState.get_conn_state_cb(conn_infos)
if (cb_state == ProcessState.NON_FUNCTIONAL):
state = ProcessState.NON_FUNCTIONAL
if description != '':
description += ' '
description += cb_description
process_status = ProcessStatus(
module_id=self.type_info._module_name, instance_id=self.instance_id,
state=ProcessStateNames[state], description=description,
connection_infos=conn_infos)
process_status_list = []
process_status_list.append(process_status)
node_status = NodeStatus(name=self.hostname,
process_status=process_status_list)
node_status_uve = NodeStatusUVE(table=self.type_info._object_table,
data=node_status)
msg = ('send_nodemgr_process_status: Sending UVE:'
+ str(node_status_uve))
self.msg_log(msg, SandeshLevel.SYS_INFO)
node_status_uve.send()
def _ksv2_project_get(self, id):
# Note: under certain circumstances (if it has been initailized
# before endpoints are populated in keystone) keystoneclient may
# be valid to list projects, but not to read them. As it won't
# be reset by resync_all_projects, it is reseted on error here.
try:
return {'name': self._ks.tenants.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.tenants.get(id).name}
def send_nodemgr_process_status_base(self, ProcessStateNames,
ProcessState, ProcessStatus):
if (self.prev_fail_status_bits != self.fail_status_bits):
self.prev_fail_status_bits = self.fail_status_bits
fail_status_bits = self.fail_status_bits
state, description = self.get_process_state(fail_status_bits)
conn_infos = ConnectionState._connection_map.values()
(cb_state, cb_description) = ConnectionState.get_conn_state_cb(conn_infos)
if (cb_state == ProcessState.NON_FUNCTIONAL):
state = ProcessState.NON_FUNCTIONAL
if description != '':
description += ' '
description += cb_description
process_status = ProcessStatus(
module_id=self.module_id, instance_id=self.instance_id,
state=ProcessStateNames[state], description=description)
process_status_list = []
process_status_list.append(process_status)
node_status = NodeStatus(name=socket.gethostname(),
process_status=process_status_list)
node_status_uve = NodeStatusUVE(table=self.table,
data=node_status)
msg = 'send_nodemgr_process_status_base: Sending UVE:' + str(node_status_uve)
self.sandesh_global.logger().log(SandeshLogger.get_py_logger_level(
SandeshLevel.SYS_INFO), msg)
node_status_uve.send()
def _health_checker(self):
while True:
try:
# do the healthcheck only if we are connected
if self._conn_state == ConnectionStatus.DOWN:
continue
meta = Metadata('display-name', '',
{'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail', elements='')
request_str = self._build_request('healthcheck', 'self', [meta])
self._publish_to_ifmap_enqueue('update', request_str, do_trace=False)
# Confirm the existence of the following default global entities in IFMAP.
search_list = [
('global-system-config', ['default-global-system-config']),
]
for type, fq_name in search_list:
if not entity_is_present(self._mapclient, type, fq_name):
raise Exception("%s not found in IFMAP DB" % ':'.join(fq_name))
# If we had unpublished the IFMAP server to discovery server earlier
# publish it back now since it is valid now.
if not self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('up', '')
self._is_ifmap_up = True
ConnectionState.update(conn_type = ConnType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.UP,
message = 'IFMAP server published',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
except Exception as e:
log_str = 'IFMAP Healthcheck failed: %s' %(str(e))
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
if self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('down',
'IFMAP DB - Invalid state')
self._is_ifmap_up = False
ConnectionState.update(conn_type = ConnType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.DOWN,
message = 'Invalid IFMAP DB State',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
finally:
gevent.sleep(
self._get_api_server().get_ifmap_health_check_interval())
def test_basic(self):
ConnectionState.init(sandesh=self._sandesh, hostname="TestHost",
module_id="TestModule", instance_id="0",
conn_status_cb=self._check_conn_status_cb,
uve_type_cls=NodeStatusUVE,
uve_data_type_cls=NodeStatus)
vcinfos = []
self._update_conn_info("Test1", ConnectionStatus.UP, "Test1 UP",
vcinfos)
self._update_conn_state("Test1", ConnectionStatus.UP, "Test1 UP",
vcinfos)
self._update_conn_info("Test2", ConnectionStatus.UP, "Test2 UP",
vcinfos)
self._update_conn_state("Test2", ConnectionStatus.UP, "Test2 UP",
vcinfos)
vcinfos = self._delete_conn_info("Test2", vcinfos)
self._delete_conn_state("Test2", vcinfos)
def _get_keystone_conn(self):
if self._ks:
return
if 'v3' in self._auth_url.split('/')[-1]:
self._ks = self._ksv3_get_conn()
if self._endpoint_type and self._ks.service_catalog:
self._ks.management_url = \
self._ks.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
else:
self._ks = self._ksv2_get_conn()
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.UP, message='',
server_addrs=[self._auth_url])
def test_callback(self):
vcinfos = []
self._update_conn_info("Test1", ConnectionStatus.UP, "Test1 UP",
vcinfos);
(pstate, message) = ConnectionState.get_conn_state_cb(vcinfos)
self.assertEqual(ProcessState.FUNCTIONAL, pstate)
self.assertEqual('', message)
self._update_conn_info("Test2", ConnectionStatus.DOWN, "Test2 DOWN",
vcinfos);
(pstate, message) = ConnectionState.get_conn_state_cb(vcinfos)
self.assertEqual(ProcessState.NON_FUNCTIONAL, pstate)
self.assertEqual("Test:Test2[Test2 DOWN] connection down", message);
self._update_conn_info("Test3", ConnectionStatus.DOWN, "Test3 DOWN",
vcinfos);
(pstate, message) = ConnectionState.get_conn_state_cb(vcinfos);
self.assertEqual(ProcessState.NON_FUNCTIONAL, pstate);
self.assertEqual("Test:Test2[Test2 DOWN], Test:Test3[Test3 DOWN] connection down", message);
def _get_keystone_conn(self):
if self._ks:
return
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'project_name': self._admin_tenant,
'username': self._auth_user,
'password': self._auth_passwd,
}
if 'v3' in self._auth_url.split('/')[-1]:
kwargs.update({
'user_domain_name': self._user_domain_name,
})
if self._user_domain_name:
kwargs.update({
'project_domain_name': self._project_domain_name,
'project_name': self._project_name,
})
else:
kwargs.update({
'domain_id': self._domain_id,
})
auth = kauth.password.Password(self._auth_url, **kwargs)
if self._use_certs:
sess = ksession.Session(auth=auth, verify=self._kscertbundle)
else:
sess = ksession.Session(auth=auth, verify=self._insecure)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.UP,
message='',
server_addrs=[self._auth_url])
def _sandesh_connection_info_update(self, status, message):
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
name = 'Zookeeper', status = new_conn_state,
message = message,
server_addrs = self._zookeeper_srvr.split(','))
if ((self._conn_state and self._conn_state != ConnectionStatus.DOWN) and
new_conn_state == ConnectionStatus.DOWN):
msg = 'Connection to Zookeeper down: %s' %(message)
self._supress_log(msg)
if (self._conn_state and self._conn_state != new_conn_state and
new_conn_state == ConnectionStatus.UP):
msg = 'Connection to Zookeeper ESTABLISHED'
self._supress_log(msg)
self._conn_state = new_conn_state
def _publish_int(self, service, data, oper_state='up', msg=''):
self.syslog('Publish service "%s", data "%s"' % (service, data))
payload = {
service: data,
'service-type': service,
'remote-addr': self.remote_addr,
'oper-state': oper_state,
'oper-state-reason': msg
}
emsg = None
cookie = None
try:
self.inc_pub_stats(service, 'request')
r = self._request(rest.OP_POST,
self.puburl,
data=json.dumps(payload),
headers=self._headers,
timeout=30)
if r.status_code != 200:
self.inc_pub_stats(service, 'sc_%d' % r.status_code)
emsg = 'Status Code ' + str(r.status_code)
except requests.exceptions.ConnectionError:
self.inc_pub_stats(service, 'conn_error')
emsg = 'Connection Error'
except requests.exceptions.Timeout:
self.inc_pub_stats(service, 'timeout')
emsg = 'Request Timeout'
finally:
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = service,
status = ConnectionStatus.DOWN if emsg else ConnectionStatus.UP,
server_addrs = ['%s:%s' % (self._server_ip, \
self._server_port)],
message = 'Publish Error - %s' % emsg if emsg else 'Publish Success')
if not emsg:
self.inc_pub_stats(service, 'response')
self.inc_pub_stats(service, 'blob', value=json.dumps(data))
response = r.json()
cookie = response['cookie']
self.pubdata[cookie] = (service, data)
self.syslog('Saving token %s' % (cookie))
return cookie
def _sandesh_connection_info_update(self, status, message):
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(conn_type=ConnectionType.ZOOKEEPER,
name=self._svc_name,
status=new_conn_state,
server_addrs=self._zk_server.split(','),
message=message)
if (self._conn_state and self._conn_state != ConnectionStatus.DOWN
and new_conn_state == ConnectionStatus.DOWN):
msg = 'Connection to Zookeeper down: %s' % (message)
self._logger.error(msg)
if (self._conn_state and self._conn_state != new_conn_state
and new_conn_state == ConnectionStatus.UP):
msg = 'Connection to Zookeeper ESTABLISHED'
self._logger.info(msg)
self._conn_state = new_conn_state
def _resync_all_projects(self):
self._get_keystone_conn()
# compare new and old set,
# optimize for common case where nothing has changed,
# so track the project-ids in a set add '-',
# keystone gives uuid without...
try:
ks_project_ids = set([
str(uuid.UUID(proj['id']))
for proj in self._ks_projects_list()
])
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %
(e, datetime.utcnow()),
server_addrs=[self._auth_url])
return True # retry
vnc_project_ids = self._vnc_project_ids
if vnc_project_ids == ks_project_ids:
# no change, go back to poll
return False
for vnc_project_id in vnc_project_ids - ks_project_ids:
self.q.put((Q_DELETE, 'project', vnc_project_id))
if self._keystone_sync_on_demand:
pass # pre_project_read will get it
else:
for ks_project_id in ks_project_ids - vnc_project_ids:
self.q.put((Q_CREATE, 'project', ks_project_id))
self.q.join()
gevent.sleep(0)
# we are in sync
self._vnc_project_ids = ks_project_ids
return False
def __init__(self, db_client_mgr, ifmap_srv_ip, ifmap_srv_port, uname,
passwd, ssl_options):
self._ifmap_srv_ip = ifmap_srv_ip
self._ifmap_srv_port = ifmap_srv_port
self._username = uname
self._password = passwd
self._ssl_options = ssl_options
self._dequeue_greenlet = None
self._CONTRAIL_XSD = "http://www.contrailsystems.com/vnc_cfg.xsd"
self._IPERMS_NAME = "id-perms"
self._NAMESPACES = {
'env': "http://www.w3.org/2003/05/soap-envelope",
'ifmap': "http://www.trustedcomputinggroup.org/2010/IFMAP/2",
'meta':
"http://www.trustedcomputinggroup.org/2010/IFMAP-METADATA/2",
'contrail': self._CONTRAIL_XSD
}
self._db_client_mgr = db_client_mgr
self._sandesh = db_client_mgr._sandesh
ConnectionState.update(
conn_type=ConnectionType.IFMAP,
name='IfMap',
status=ConnectionStatus.INIT,
message='',
server_addrs=["%s:%s" % (ifmap_srv_ip, ifmap_srv_port)])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
self._queue = Queue(self._get_api_server()._args.ifmap_queue_size)
self.reset()
# Set the signal handler
signal.signal(signal.SIGUSR2, self.handler)
self._init_conn()
self._publish_config_root()
self._health_checker_greenlet =\
vnc_greenlets.VncGreenlet('VNC IfMap Health Checker',
self._health_checker)
def sandesh_init(self):
""" Init Sandesh """
self._sandesh = Sandesh()
# Reset sandesh send rate limit value.
if self._args.sandesh_send_rate_limit is not None:
SandeshSystem.set_sandesh_send_rate_limit(
self._args.sandesh_send_rate_limit)
# Register custom sandesh request handlers.
self._redefine_sandesh_handles()
# Initialize Sandesh generator.
self._sandesh.init_generator(
self._module["name"],
self._module["hostname"],
self._module["node_type_name"],
self._module["instance_id"],
self._args.collectors,
'kube_manager_context',
int(self._args.http_server_port), ['cfgm_common', 'kube_manager'],
self._module["discovery"],
logger_class=self._args.logger_class,
logger_config_file=self._args.logging_conf,
config=self._args.sandesh_config)
# Set Sandesh logging params.
self._sandesh.set_logging_params(
enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level,
file=self._args.log_file,
enable_syslog=self._args.use_syslog,
syslog_facility=self._args.syslog_facility)
# Connect to collector.
ConnectionState.init(
self._sandesh, self._module["hostname"], self._module["name"],
self._module["instance_id"],
staticmethod(ConnectionState.get_process_state_cb), NodeStatusUVE,
NodeStatus, self._module["table"])
def __init__(self, redis_uve_list, logger,
redis_password=None, \
uvedbcache=None, usecache=False, freq=5):
self._logger = logger
self._redis = None
self._uvedbcache = uvedbcache
self._usecache = usecache
self._redis_password = redis_password
self._uve_reverse_map = {}
self._freq = freq
for h,m in UVE_MAP.iteritems():
self._uve_reverse_map[m] = h
# Fill in redis/collector instances
self._redis_uve_map = {}
for new_elem in redis_uve_list:
test_elem = RedisInstKey(ip=new_elem[0], port=new_elem[1])
self._redis_uve_map[test_elem] = RedisInst()
ConnectionState.update(ConnectionType.REDIS_UVE,\
test_elem.ip+":"+str(test_elem.port), ConnectionStatus.INIT)
def _cassandra_init_conn_pools(self):
for ks,cf_list in self._keyspaces.items():
pool = pycassa.ConnectionPool(
ks, self._server_list, max_overflow=-1, use_threadlocal=True,
prefill=True, pool_size=20, pool_timeout=120,
max_retries=-1, timeout=5, credentials=self._credential)
rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
for (cf, _) in cf_list:
self._cf_dict[cf] = ColumnFamily(
pool, cf, read_consistency_level = rd_consistency,
write_consistency_level = wr_consistency)
ConnectionState.update(conn_type = ConnectionType.DATABASE,
name = 'Cassandra', status = ConnectionStatus.UP, message = '',
server_addrs = self._server_list)
self._conn_state = ConnectionStatus.UP
msg = 'Cassandra connection ESTABLISHED'
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def update_redis_uve_list(self, redis_uve_list):
newlist = set(redis_uve_list)
# if some redis instances are gone, remove them from our map
for test_elem in self._redis_uve_map.keys():
r_ip = test_elem[0]
r_port = test_elem[1]
redis_inst = (r_ip, int(r_port))
if redis_inst not in newlist:
self._redis_uve_map[test_elem].deleted = True
else:
self._redis_uve_map[test_elem].deleted = False
# new redis instances need to be inserted into the map
for new_elem in newlist:
new_redis = RedisInstKey(ip=new_elem[0], port=new_elem[1])
if new_redis not in self._redis_uve_map:
self._redis_uve_map[new_redis] = RedisInst()
ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,\
name = new_elem[0]+":"+str(new_elem[1]), status = \
ConnectionStatus.INIT, server_addrs = \
[new_elem[0]+":"+str(new_elem[1])])