def _cassandra_init_conn_pools(self):
for ks, cf_list in self._keyspaces.items():
pool = pycassa.ConnectionPool(
ks,
self._server_list,
max_overflow=-1,
use_threadlocal=True,
prefill=True,
pool_size=20,
pool_timeout=120,
max_retries=-1,
timeout=5,
)
rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
for (cf, _) in cf_list:
self._cf_dict[cf] = ColumnFamily(
pool, cf, read_consistency_level=rd_consistency, write_consistency_level=wr_consistency
)
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="Cassandra",
status=ConnectionStatus.UP,
message="",
server_addrs=self._server_list,
)
self._conn_state = ConnectionStatus.UP
msg = "Cassandra connection ESTABLISHED"
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _ksv3_project_get(self, id=None, name=None):
if id:
try:
project = self._ks.projects.get(id)
return {
'id': project.id,
'name': project.name,
'domain_id': project.domain_id
}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %
(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
project = self._ks.projects.get(id)
return {
'id': project.id,
'name': project.name,
'domain_id': project.domain_id
}
else:
id = None
for tenant in self._ks.projects.list():
if tenant.name == name:
id = tenant.id
break
return {'name': name, 'id': id}
def _cassandra_init_conn_pools(self):
socket_factory = self._make_socket_factory()
for ks, cf_dict in itertools.chain(list(self._rw_keyspaces.items()),
list(self._ro_keyspaces.items())):
keyspace = '%s%s' % (self._db_prefix, ks)
pool = pycassa.ConnectionPool(keyspace,
self._server_list,
max_overflow=5,
use_threadlocal=True,
prefill=True,
pool_size=self._pool_size,
pool_timeout=120,
max_retries=15,
timeout=5,
credentials=self._credential,
socket_factory=socket_factory)
for cf_name in cf_dict:
cf_kwargs = cf_dict[cf_name].get('cf_args', {})
self._cf_dict[cf_name] = ColumnFamily(
pool,
cf_name,
read_consistency_level=ConsistencyLevel.QUORUM,
write_consistency_level=ConsistencyLevel.QUORUM,
dict_class=dict,
**cf_kwargs)
ConnectionState.update(conn_type=ConnType.DATABASE,
name='Cassandra',
status=ConnectionStatus.UP,
message='',
server_addrs=self._server_list)
self._conn_state = ConnectionStatus.UP
msg = 'Cassandra connection ESTABLISHED'
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _redis_inst_up(self, r_inst, redish):
if not self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = redish
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.UP)
def _publish_to_ifmap(self, oper_body):
try:
not_published = True
retry_count = 0
resp_xml = None
while not_published:
sess_id = self._mapclient.get_session_id()
req_xml = PublishRequest(sess_id, oper_body)
resp_xml = self._mapclient.call('publish', req_xml)
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if err_codes:
if retry_count == 0:
log_str = 'Error publishing to ifmap, req: %s, resp: %s' \
%(req_xml, resp_xml)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
ConnectionState.update(
conn_type = ConnType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.INIT,
message = 'Session lost, renew it',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
retry_count = retry_count + 1
self._init_conn()
if self._ifmap_restarted():
msg = "IF-MAP servers restarted, re-populate it"
self.config_log(msg, level=SandeshLevel.SYS_ERR)
self.reset()
self._get_api_server().publish_ifmap_to_discovery(
'down', msg)
self._publish_config_root()
self._db_client_mgr.db_resync()
self._publish_to_ifmap_enqueue('publish_discovery', 1)
else: # successful publish
not_published = False
break
# end while not_published
if retry_count:
log_str = 'Success publishing to ifmap after %d tries' \
%(retry_count)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
return True, resp_xml
except Exception as e:
# Failed to publish the operation due to unknown error.
# Probably a connection issue with the ifmap server.
msg = "Failed to publish request %s: %s" % (oper_body, str(e))
return False, msg
def _cassandra_init_conn_pools(self):
for ks, cf_list in self._keyspaces.items():
pool = pycassa.ConnectionPool(ks,
self._server_list,
max_overflow=-1,
use_threadlocal=True,
prefill=True,
pool_size=20,
pool_timeout=120,
max_retries=-1,
timeout=5)
rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM
for (cf, _) in cf_list:
self._cf_dict[cf] = ColumnFamily(
pool,
cf,
read_consistency_level=rd_consistency,
write_consistency_level=wr_consistency)
ConnectionState.update(conn_type=ConnectionType.DATABASE,
name='Cassandra',
status=ConnectionStatus.UP,
message='',
server_addrs=self._server_list)
self._conn_state = ConnectionStatus.UP
msg = 'Cassandra connection ESTABLISHED'
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _query(self):
conn_state_updated = False
# hoping all errors are transient and a little wait will solve the problem
while True:
try:
self.stats['request'] += 1
r = requests.post(self.url,
data=self.post_body,
headers=self._headers,
timeout=5)
if r.status_code == 200:
break
self.inc_stats('sc_%d' % r.status_code)
emsg = "Status Code %d" % r.status_code
except requests.exceptions.ConnectionError:
self.stats['conn_error'] += 1
emsg = 'Connection Error'
except (requests.exceptions.Timeout, socket.timeout):
self.stats['timeout'] += 1
emsg = 'Request Timeout'
self.syslog('connection error or failed to subscribe')
if not conn_state_updated:
conn_state_updated = True
ConnectionState.update(
conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.DOWN,
message = 'Subscribe - %s' % emsg,
server_addrs = \
['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
gevent.sleep(2)
# end while
self.syslog('query resp => %s ' % r.text)
response = r.json()
# avoid signature on ttl which can change between iterations
info = response[self.service_type]
infostr = json.dumps(info)
sig = hashlib.md5(infostr).hexdigest()
self.stats['response'] += 1
self.stats['ttl'] = response['ttl']
self.stats['blob'] = infostr
self.ttl = response['ttl']
self.change = False
if sig != self.sig:
#print 'signature mismatch! old=%s, new=%s' % (self.sig, sig)
self.info = info
self.sig = sig
self.change = True
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.UP,
message = 'Subscribe Response',
server_addrs = ['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
def _ksv2_project_get(self, id=None, name=None):
# Note: under certain circumstances (if it has been initailized
# before endpoints are populated in keystone) keystoneclient may
# be valid to list projects, but not to read them. As it won't
# be reset by resync_all_projects, it is reseted on error here.
if id:
try:
return {'name': self._ks.tenants.get(id).name, 'id': id}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %
(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.tenants.get(id).name, 'id': id}
else:
id = None
for tenant in self._ks.tenants.list():
if tenant.name == name:
id = tenant.id
break
return {'name': name, 'id': id}
def __init__(self, redis_uve_list, logger,
redis_password=None, redis_ssl_params=None, \
uvedbcache=None, usecache=False, freq=5):
self._logger = logger
self._redis = None
self._uvedbcache = uvedbcache
self._usecache = usecache
self._redis_cfg_info = []
self._redis_password = redis_password
self._redis_ssl_params = redis_ssl_params
self._uve_reverse_map = {}
self._freq = freq
self._active_collectors = []
for h, m in UVE_MAP.items():
self._uve_reverse_map[m] = h
# Fill in redis/collector instances
self._redis_uve_map = {}
for new_elem in redis_uve_list:
test_elem = RedisInstKey(ip=new_elem[0], port=new_elem[1])
self._redis_cfg_info.append(test_elem)
self._redis_uve_map[test_elem] = RedisInst()
ConnectionState.update(ConnectionType.REDIS_UVE,\
test_elem.ip+":"+str(test_elem.port), ConnectionStatus.INIT,
[test_elem.ip+":"+str(test_elem.port)],"Redis Instance initialized")
def update_redis_uve_list(self, redis_uve_list):
newlist = set(redis_uve_list)
chg = False
# if some redis instances are gone, remove them from our map
for test_elem in self._redis_uve_map.keys():
if test_elem not in newlist:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
del self._redis_uve_map[test_elem]
ConnectionState.delete(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port))
# new redis instances need to be inserted into the map
for test_elem in newlist:
if test_elem not in self._redis_uve_map:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
self._redis_uve_map[test_elem] = None
ConnectionState.update(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port), ConnectionStatus.INIT)
if chg:
self._logger.error("updated redis_uve_list %s" % str(self._redis_uve_map))
# Exercise redis connections to update health
if len(newlist):
self.get_uve("ObjectCollectorInfo:__NONE__", False, None)
def heartbeat(self):
while True:
# send heartbeat for each published object seperately
# dictionary can change size during iteration
pub_list = self.pubdata.copy()
for cookie in pub_list:
payload = {'cookie': cookie}
self.syslog('Sending cookie %s in heartbeat' % cookie)
try:
r = requests.post(
self.hburl, data=json.dumps(payload), headers=self._headers)
except requests.exceptions.ConnectionError:
service, data = pub_list[cookie]
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = service, status = ConnectionStatus.DOWN,
server_addrs = ['%s:%s' % (self._server_ip, \
self._server_port)],
message = 'HeartBeat - Connection Error')
self.syslog('Connection Error')
continue
# if DS lost track of our data, republish
if r.status_code == 404:
# forget cached cookie and object; will re-learn
self.syslog('Server lost track of token %s' % (cookie))
service, data = self.pubdata[cookie]
del self.pubdata[cookie]
self._publish_int(service, data)
gevent.sleep(HC_INTERVAL)
def update_redis_uve_list(self, redis_uve_list):
newlist = set(redis_uve_list)
chg = False
# if some redis instances are gone, remove them from our map
for test_elem in self._redis_uve_map.keys():
if test_elem not in newlist:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
del self._redis_uve_map[test_elem]
ConnectionState.delete(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port))
# new redis instances need to be inserted into the map
for test_elem in newlist:
if test_elem not in self._redis_uve_map:
chg = True
r_ip = test_elem[0]
r_port = test_elem[1]
self._redis_uve_map[test_elem] = None
ConnectionState.update(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port), ConnectionStatus.INIT)
if chg:
self._logger.error("updated redis_uve_list %s" %
str(self._redis_uve_map))
# Exercise redis connections to update health
if len(newlist):
self.get_uve("ObjectCollectorInfo:__NONE__", False, None)
def conn_state_notify(self, svc, msg='', up=True, servers=''):
ctype = self.map_svc(svc)
status = self.map_sts(up)
if isinstance(servers, basestring):
servers = [servers]
ConnectionState.update(conn_type=ctype, name='SNMP', status=status,
message=msg, server_addrs=servers)
def _redis_inst_down(self, r_inst):
if self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = None
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.DOWN)
def _redis_inst_up(self, r_inst, redish):
if not self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = redish
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.UP)
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnType.APISERVER, name='ApiServer',
status=status,
message=message or 'ApiServer Connection State updated',
server_addrs=['%s:%s' % (self._args.api_server_ip,
self._args.api_server_port)])
def heartbeat(self):
while True:
# send heartbeat for each published object seperately
# dictionary can change size during iteration
pub_list = self.pubdata.copy()
for cookie in pub_list:
payload = {'cookie': cookie}
self.syslog('Sending cookie %s in heartbeat' % cookie)
try:
r = requests.post(
self.hburl, data=json.dumps(payload), headers=self._headers)
except requests.exceptions.ConnectionError:
service, data = pub_list[cookie]
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = service, status = ConnectionState.DOWN,
server_addrs = ['%s:%s' % (self._server_ip, \
self._server_port)],
message = 'HeartBeat - Connection Error')
self.syslog('Connection Error')
continue
# if DS lost track of our data, republish
if r.status_code == 404:
# forget cached cookie and object; will re-learn
self.syslog('Server lost track of token %s' % (cookie))
service, data = self.pubdata[cookie]
del self.pubdata[cookie]
self._publish_int(service, data)
gevent.sleep(HC_INTERVAL)
def _redis_inst_down(self, r_inst):
if self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = None
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port), ConnectionStatus.DOWN)
def vnc_api(self):
e = SystemError('Cant connect to API server')
api_servers = [srv.split(':')[0] for srv in self._args.api_server_list]
api_server_port = self._args.api_server_list[0].split(':')[1] \
if self._args.api_server_list else None
try:
vnc = VncApi(self._args.admin_user,
self._args.admin_password,
self._args.admin_tenant_name,
api_servers,
api_server_port,
api_server_use_ssl=self._args.api_server_use_ssl,
auth_host=self._args.auth_host,
auth_port=self._args.auth_port,
auth_protocol=self._args.auth_protocol)
except Exception as e:
ConnectionState.update(conn_type=ConnectionType.APISERVER,
name='Config',
status=ConnectionStatus.DOWN,
message=str(e),
server_addrs=api_servers)
return None
else:
ConnectionState.update(conn_type=ConnectionType.APISERVER,
name='Config',
status=ConnectionStatus.UP,
server_addrs=api_servers)
return vnc
def _update_sandesh_status(self, status, msg=''):
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name='RabbitMQ',
status=status,
message=msg,
server_addrs=["%s:%s" % (self._rabbit_ip, self._rabbit_port)])
def _publish_to_ifmap(self, oper_body):
try:
not_published = True
retry_count = 0
resp_xml = None
while not_published:
sess_id = self._mapclient.get_session_id()
req_xml = PublishRequest(sess_id, oper_body)
resp_xml = self._mapclient.call('publish', req_xml)
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if err_codes:
if retry_count == 0:
log_str = 'Error publishing to ifmap, req: %s, resp: %s' \
%(req_xml, resp_xml)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
ConnectionState.update(
conn_type = ConnectionType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.INIT,
message = 'Session lost, renew it',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
retry_count = retry_count + 1
self._init_conn()
if self._ifmap_restarted():
msg = "IF-MAP servers restarted, re-populate it"
self.config_log(msg, level=SandeshLevel.SYS_ERR)
self.reset()
self._get_api_server().publish_ifmap_to_discovery(
'down', msg)
self._publish_config_root()
self._db_client_mgr.db_resync()
self._publish_to_ifmap_enqueue('publish_discovery', 1)
else: # successful publish
not_published = False
break
# end while not_published
if retry_count:
log_str = 'Success publishing to ifmap after %d tries' \
%(retry_count)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
return True, resp_xml
except Exception as e:
# Failed to publish the operation due to unknown error.
# Probably a connection issue with the ifmap server.
msg = "Failed to publish request %s: %s" % (oper_body, str(e))
return False, msg
def run(self):
lck = False
while True:
try:
k, value = self._redis.brpop("DELETED")
self._sem.acquire()
lck = True
self._logger.debug("%s del received for " % value)
# value is of the format:
# DEL:<key>:<src>:<node-type>:<module>:<instance-id>:<message-type>:<seqno>
self._redis.delete(value)
except redis.exceptions.ResponseError:
#send redis connection down msg. Coule be bcos of authentication
ConnectionState.update(conn_type = ConnectionType.REDIS,
name = 'UVE', status = ConnectionStatus.DOWN,
message = 'UVE result : Connection Error',
server_addrs = ['%s:%d' % (self._local_redis_uve[0],
self._local_redis_uve[1])])
sys.exit()
except redis.exceptions.ConnectionError:
if lck:
self._sem.release()
lck = False
gevent.sleep(5)
else:
if lck:
self._sem.release()
lck = False
self._logger.debug("Deleted %s" % value)
self._logger.debug("UVE %s Type %s" % (key, typ))
def conn_state_notify(self, svc, msg='', up=True, servers=''):
ctype = self.map_svc(svc)
status = self.map_sts(up)
if isinstance(servers, basestring):
servers = [servers]
ConnectionState.update(conn_type=ctype, name='SNMP', status=status,
message=msg, server_addrs=servers)
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnType.APISERVER, name='ApiServer',
status=status,
message=message or 'ApiServer Connection State updated',
server_addrs=['%s:%s' % (self._args.api_server_ip,
self._args.api_server_port)])
def run(self):
exitrun = False
while not exitrun:
for rkey in self._redis_uve_map.keys():
rinst = self._redis_uve_map[rkey]
old_pid = rinst.collector_pid
try:
# check if it is marked as deleted during sighup handling
if rinst.deleted == True:
r_ip = rkey[0]
r_port = rkey[1]
del self._redis_uve_map[rkey]
ConnectionState.delete(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port))
continue
if rinst.redis_handle is None:
rinst.redis_handle = StrictRedisWrapper(
host=rkey.ip,
port=rkey.port,
password=self._redis_password,
db=1,
socket_timeout=30)
rinst.collector_pid = None
# check for known collector pid string
# if there's a mismatch, we must read it again
if rinst.collector_pid is not None:
if not rinst.redis_handle.sismember(
"NGENERATORS", rinst.collector_pid):
rinst.collector_pid = None
# read the collector pid string
if rinst.collector_pid is None:
for gen in rinst.redis_handle.smembers("NGENERATORS"):
module = gen.split(':')[2]
if module == "contrail-collector":
rinst.collector_pid = gen
except gevent.GreenletExit:
self._logger.error('UVEServer Exiting on gevent-kill')
exitrun = True
break
except Exception as e:
self._logger.error("redis/collector healthcheck failed %s for %s" \
% (str(e), str(rkey)))
rinst.redis_handle = None
rinst.collector_pid = None
finally:
# Update redis/collector health
if old_pid is None and rinst.collector_pid is not None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.UP,
[rkey.ip+":"+str(rkey.port)])
if old_pid is not None and rinst.collector_pid is None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.DOWN,
[rkey.ip+":"+str(rkey.port)])
if not exitrun:
gevent.sleep(self._freq)
def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh):
global openstack_driver
openstack_driver = self
if api_server_ip == '0.0.0.0':
self._vnc_api_ip = '127.0.0.1'
else:
self._vnc_api_ip = api_server_ip
self._vnc_api_port = api_server_port
self._config_sections = conf_sections
fill_keystone_opts(self, conf_sections)
if 'v3' in self._auth_url.split('/')[-1]:
self._ks_domains_list = self._ksv3_domains_list
self._ks_domain_get = self._ksv3_domain_get
self._ks_projects_list = self._ksv3_projects_list
self._ks_project_get = self._ksv3_project_get
self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc
self._add_project_to_vnc = self._ksv3_add_project_to_vnc
self._del_project_from_vnc = self._ksv3_del_project_from_vnc
self._vnc_default_domain_id = None
else:
self._ks_domains_list = None
self._ks_domain_get = None
self._ks_projects_list = self._ksv2_projects_list
self._ks_project_get = self._ksv2_project_get
self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc
self._add_project_to_vnc = self._ksv2_add_project_to_vnc
self._del_project_from_vnc = self._ksv2_del_project_from_vnc
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.INIT, message='',
server_addrs=[self._auth_url])
self._vnc_lib = None
# resync failures, don't retry forever
self._failed_domain_dels = set()
self._failed_project_dels = set()
# active domains/projects in contrail/vnc api server
self._vnc_domain_ids = set()
self._vnc_project_ids = set()
# logging
self._sandesh_logger = sandesh.logger()
self._vnc_os_logger = logging.getLogger(__name__)
self._vnc_os_logger.setLevel(logging.ERROR)
# Add the log message handler to the logger
try:
with open(self._err_file, 'a'):
handler = logging.handlers.RotatingFileHandler(
self._err_file, maxBytes=64*1024, backupCount=5)
self._vnc_os_logger.addHandler(handler)
except IOError:
self._sandesh_logger.error("Failed to open trace file %s" %
self._err_file)
self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh):
global openstack_driver
openstack_driver = self
if api_server_ip == '0.0.0.0':
self._vnc_api_ip = '127.0.0.1'
else:
self._vnc_api_ip = api_server_ip
self._vnc_api_port = api_server_port
self._config_sections = conf_sections
fill_keystone_opts(self, conf_sections)
if 'v3' in self._auth_url.split('/')[-1]:
self._ks_domains_list = self._ksv3_domains_list
self._ks_domain_get = self._ksv3_domain_get
self._ks_projects_list = self._ksv3_projects_list
self._ks_project_get = self._ksv3_project_get
self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc
self._add_project_to_vnc = self._ksv3_add_project_to_vnc
self._del_project_from_vnc = self._ksv3_del_project_from_vnc
self._vnc_default_domain_id = None
else:
self._ks_domains_list = None
self._ks_domain_get = None
self._ks_projects_list = self._ksv2_projects_list
self._ks_project_get = self._ksv2_project_get
self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc
self._add_project_to_vnc = self._ksv2_add_project_to_vnc
self._del_project_from_vnc = self._ksv2_del_project_from_vnc
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.INIT, message='',
server_addrs=[self._auth_url])
self._vnc_lib = None
# resync failures, don't retry forever
self._failed_domain_dels = set()
self._failed_project_dels = set()
# active domains/projects in contrail/vnc api server
self._vnc_domain_ids = set()
self._vnc_project_ids = set()
# logging
self._sandesh_logger = sandesh.logger()
self._vnc_os_logger = logging.getLogger(__name__)
self._vnc_os_logger.setLevel(logging.ERROR)
# Add the log message handler to the logger
try:
with open(self._err_file, 'a'):
handler = logging.handlers.RotatingFileHandler(
self._err_file, maxBytes=64*1024, backupCount=5)
self._vnc_os_logger.addHandler(handler)
except IOError:
self._sandesh_logger.error("Failed to open trace file %s" %
self._err_file)
self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
def _redis_inst_up(self, r_inst, redish):
if r_inst in self._redis_uve_map and not self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = redish
ConnectionState.update(ConnectionType.REDIS_UVE,\
r_ip + ":" + str(r_port), ConnectionStatus.UP,
[r_ip + ":" + str(r_port)], message = 'Redis Instance is up')
def _update_sandesh_status(self, status, msg=''):
ConnectionState.update(conn_type=ConnType.DATABASE,
name='Cassandra',
status=status,
message=msg,
server_addrs=self._server_list)
# Keeps trace of the current status.
self._conn_state = status
def _update_sandesh_status(self, status, msg=""):
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="Cassandra",
status=status,
message=msg,
server_addrs=self._server_list,
)
def _update_sandesh_status(self, status, msg=""):
ConnectionState.update(
conn_type=ConnectionType.DATABASE,
name="RabbitMQ",
status=status,
message=msg,
server_addrs=["%s:%s" % (self._rabbit_ip, self._rabbit_port)],
)
def _update_connection_state(self, status, message=''):
server_addrs = ['%s:%d' % (self._conf_info['api_server_ip'], \
self._conf_info['api_server_port'])]
ConnectionState.update(conn_type=ConnectionType.APISERVER,
name='',
status=status,
message=message,
server_addrs=server_addrs)
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnectionType.APISERVER,
name="ApiServer",
status=status,
message=message or "",
server_addrs=["%s:%s" % (self._args.api_server_ip, self._args.api_server_port)],
)
def _sandesh_connection_info_update(self, status, message):
from pysandesh.connection_info import ConnectionState
from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \
ConnectionType
ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
name = 'Zookeeper', status = getattr(ConnectionStatus, status),
message = message,
server_addrs = self._server_list.split(','))
def _health_checker(self):
while True:
try:
# do the healthcheck only if we are connected
if self._conn_state == ConnectionStatus.DOWN:
continue
meta = Metadata('display-name',
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements='')
request_str = self._build_request('healthcheck', 'self',
[meta])
self._publish_to_ifmap_enqueue('update',
request_str,
do_trace=False)
# Confirm the existence of the following default global entities in IFMAP.
search_list = [
('global-system-config', ['default-global-system-config']),
]
for type, fq_name in search_list:
if not entity_is_present(self._mapclient, type, fq_name):
raise Exception("%s not found in IFMAP DB" %
':'.join(fq_name))
# If we had unpublished the IFMAP server to discovery server earlier
# publish it back now since it is valid now.
if not self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('up', '')
self._is_ifmap_up = True
ConnectionState.update(
conn_type=ConnectionType.IFMAP,
name='IfMap',
status=ConnectionStatus.UP,
message='',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
except Exception as e:
log_str = 'IFMAP Healthcheck failed: %s' % (str(e))
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
if self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery(
'down', 'IFMAP DB - Invalid state')
self._is_ifmap_up = False
ConnectionState.update(
conn_type=ConnectionType.IFMAP,
name='IfMap',
status=ConnectionStatus.DOWN,
message='Invalid IFMAP DB State',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
finally:
gevent.sleep(
self._get_api_server().get_ifmap_health_check_interval())
def _query(self):
conn_state_updated = False
# hoping all errors are transient and a little wait will solve the problem
while True:
try:
self.stats['request'] += 1
r = requests.post(
self.url, data=self.post_body, headers=self._headers, timeout=5)
if r.status_code == 200:
break
self.inc_stats('sc_%d' % r.status_code)
emsg = "Status Code %d" % r.status_code
except requests.exceptions.ConnectionError:
self.stats['conn_error'] += 1
emsg = 'Connection Error'
except (requests.exceptions.Timeout, socket.timeout):
self.stats['timeout'] += 1
emsg = 'Request Timeout'
self.syslog('connection error or failed to subscribe')
if not conn_state_updated:
conn_state_updated = True
ConnectionState.update(
conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.DOWN,
message = 'Subscribe - %s' % emsg,
server_addrs = \
['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
gevent.sleep(2)
# end while
self.syslog('query resp => %s ' % r.text)
response = r.json()
# avoid signature on ttl which can change between iterations
info = response[self.service_type]
infostr = json.dumps(info)
sig = hashlib.md5(infostr).hexdigest()
self.stats['response'] += 1
self.stats['ttl'] = response['ttl']
self.stats['blob'] = infostr
self.ttl = response['ttl']
self.change = False
if sig != self.sig:
#print 'signature mismatch! old=%s, new=%s' % (self.sig, sig)
self.info = info
self.sig = sig
self.change = True
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.UP,
message = 'Subscribe Response',
server_addrs = ['%s:%s' % (self.dc._server_ip, \
self.dc._server_port)])
def __init__(self, dc, service_type, count, f=None, *args, **kw):
super(Subscribe, self).__init__()
self.verify = kw.pop("verify")
self.protocol = "http"
if self.verify:
self.protocol = "https"
self.dc = dc
self.f = f
self.kw = kw
self.args = args
self.count = count
self.service_type = service_type
self._headers = {
'Content-type': 'application/json',
}
self.info = []
infostr = json.dumps(self.info)
self.sig = hashlib.md5(infostr).hexdigest()
self.done = False
self.stats = {
'service_type': service_type,
'request': 0,
'response': 0,
'conn_error': 0,
'timeout': 0,
'exc_unknown': 0,
'exc_info': '',
'instances': count,
'ttl': 0,
'blob': '',
}
self.data = {
'service': service_type,
'instances': count,
'client-type': dc._client_type,
'remote-addr': dc.remote_addr,
'client': dc._myid
}
self.post_body = json.dumps(self.data)
self.url = "%s://%s:%s/subscribe" % (self.protocol, dc._server_ip,
dc._server_port)
if f:
# asynch - callback when new info is received
ConnectionState.update(
conn_type=ConnectionType.DISCOVERY,
name=self.service_type,
status=ConnectionStatus.INIT,
message='Subscribe',
server_addrs=['%s:%s' % (dc._server_ip, dc._server_port)])
self.task = gevent.spawn(self.ttl_loop)
else:
self._query()
self.done = True
def _redis_inst_down(self, r_inst):
if r_inst in self._redis_uve_map and self._redis_uve_map[r_inst]:
r_ip = r_inst[0]
r_port = r_inst[1]
self._redis_uve_map[r_inst] = None
ConnectionState.update(ConnectionType.REDIS_UVE,
r_ip + ":" + str(r_port),
ConnectionStatus.DOWN,
message='Redis Instance is down')
def run(self):
exitrun = False
while not exitrun:
for rkey in self._redis_uve_map.keys():
rinst = self._redis_uve_map[rkey]
old_pid = rinst.collector_pid
try:
# check if it is marked as deleted during sighup handling
if rinst.deleted == True:
r_ip = rkey[0]
r_port = rkey[1]
del self._redis_uve_map[rkey]
ConnectionState.delete(ConnectionType.REDIS_UVE,\
r_ip+":"+str(r_port))
continue
if rinst.redis_handle is None:
rinst.redis_handle = StrictRedisWrapper(
host=rkey.ip, port=rkey.port,
password=self._redis_password, db=1,
socket_timeout=30)
rinst.collector_pid = None
# check for known collector pid string
# if there's a mismatch, we must read it again
if rinst.collector_pid is not None:
if not rinst.redis_handle.sismember("NGENERATORS", rinst.collector_pid):
rinst.collector_pid = None
# read the collector pid string
if rinst.collector_pid is None:
for gen in rinst.redis_handle.smembers("NGENERATORS"):
module = gen.split(':')[2]
if module == "contrail-collector":
rinst.collector_pid = gen
except gevent.GreenletExit:
self._logger.error('UVEServer Exiting on gevent-kill')
exitrun = True
break
except Exception as e:
self._logger.error("redis/collector healthcheck failed %s for %s" \
% (str(e), str(rkey)))
rinst.redis_handle = None
rinst.collector_pid = None
finally:
# Update redis/collector health
if old_pid is None and rinst.collector_pid is not None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.UP,
[rkey.ip+":"+str(rkey.port)])
if old_pid is not None and rinst.collector_pid is None:
ConnectionState.update(ConnectionType.REDIS_UVE,\
rkey.ip + ":" + str(rkey.port), ConnectionStatus.DOWN,
[rkey.ip+":"+str(rkey.port)])
if not exitrun:
gevent.sleep(self._freq)
def _get_keystone_conn(self):
if self._ks:
return
if self._use_certs:
verify = self._kscertbundle
else:
if self._insecure:
verify = False
else:
verify = True
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'project_name': self._admin_tenant,
'username': self._auth_user,
'password': self._auth_passwd,
'user_domain_name': self._user_domain_name,
}
if self._user_domain_name:
kwargs.update({
'project_domain_name': self._project_domain_name,
'project_name': self._project_name,
})
else:
kwargs.update({
'domain_id': self._domain_id,
})
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
try:
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
except kexceptions.DiscoveryFailure:
# Probably a v2 Keytone API, remove v3 args and try again
v3_args = ['user_domain_name', 'project_domain_name', 'domain_id']
for arg in v3_args:
kwargs.pop(arg, None)
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.UP,
message='',
server_addrs=[self._auth_url])
def connection_state_update(self, status, message=None):
ConnectionState.update(
conn_type=ConnType.APISERVER,
name='ApiServer',
status=status,
message=message or '',
server_addrs=[
'%s:%s' %
(self.args.vnc_endpoint_ip, self.args.vnc_endpoint_port)
])
def _get_keystone_conn(self):
if self._ks:
return
if self._use_certs:
verify=self._kscertbundle
else:
if self._insecure:
verify=False
else:
verify=True
if self._admin_token:
auth = kauth.token.Token(self._auth_url, token=self._admin_token)
else:
kwargs = {
'project_name': self._admin_tenant,
'username': self._auth_user,
'password': self._auth_passwd,
'user_domain_name': self._user_domain_name,
}
if self._user_domain_name:
kwargs.update({
'project_domain_name': self._project_domain_name,
'project_name': self._project_name,
})
else:
kwargs.update({
'domain_id': self._domain_id,
})
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
try:
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
except kexceptions.DiscoveryFailure:
# Probably a v2 Keytone API, remove v3 args and try again
v3_args = ['user_domain_name', 'project_domain_name', 'domain_id']
for arg in v3_args:
kwargs.pop(arg, None)
auth = kauth.password.Password(self._auth_url, **kwargs)
sess = ksession.Session(auth=auth, verify=verify)
self._ks = kclient.Client(session=sess, auth_url=self._auth_url)
if self._endpoint_type and auth.auth_ref.service_catalog:
self._ks.management_url = \
auth.auth_ref.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone',
status=ConnectionStatus.UP, message='',
server_addrs=[self._auth_url])
def connection_state_update(status, message=None):
ConnectionState.update(conn_type=ConnType.APISERVER,
name='ApiServer',
status=status,
message=message or '',
server_addrs=[
'%s:%s' %
(args.api_server_ip, args.api_server_port)
])
st_logger.notice(
"############# UPDATE CONNECTION STATE: {}".format(status))
def _ksv3_domain_get(self, id=None):
try:
return {'name': self._ks.domains.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.domains.get(id).name}
def _ksv3_domain_get(self, id=None):
try:
return {'name': self._ks.domains.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.domains.get(id).name}
def _init_conn(self):
self._mapclient = client(
("%s" % (self._ifmap_srv_ip), "%s" % (self._ifmap_srv_port)),
self._username, self._password, self._NAMESPACES,
self._ssl_options)
connected = False
while not connected:
try:
resp_xml = self._mapclient.call('newSession',
NewSessionRequest())
except socket.error as e:
msg = 'Failed to establish IF-MAP connection: %s' % str(e)
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
continue
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if not err_codes:
connected = True
else:
msg = "Failed to establish IF-MAP connection: %s" % err_codes
self.config_log(msg, level=SandeshLevel.SYS_WARN)
session_id = self._mapclient.get_session_id()
try:
self._mapclient.call('endSession',
EndSessionRequest(session_id))
except socket.error as e:
msg = "Failed to end the IF-MAP session %s: %s" %\
(session_id, str(e))
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
ConnectionState.update(conn_type=ConnectionType.IFMAP,
name='IfMap',
status=ConnectionStatus.UP,
message='',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
self._conn_state = ConnectionStatus.UP
msg = 'IFMAP connection ESTABLISHED'
self.config_log(msg, level=SandeshLevel.SYS_NOTICE)
self._mapclient.set_session_id(
newSessionResult(resp_xml).get_session_id())
self._mapclient.set_publisher_id(
newSessionResult(resp_xml).get_publisher_id())
def __init__(self, dc, service_type, count, f=None, *args, **kw):
self.dc = dc
self.f = f
self.kw = kw
self.args = args
self.count = count
self.service_type = service_type
self._headers = {
'Content-type': 'application/json',
}
self.info = []
infostr = json.dumps(self.info)
self.sig = hashlib.md5(infostr).hexdigest()
self.done = False
self.stats = {
'service_type' : service_type,
'request' : 0,
'response' : 0,
'conn_error' : 0,
'timeout' : 0,
'exc_unknown' : 0,
'exc_info' : '',
'instances' : count,
'ttl' : 0,
'blob' : '',
}
data = {
'service': service_type,
'instances': count,
'client-type': dc._client_type,
'remote-addr': dc._myip,
'client': dc._myid
}
self.post_body = json.dumps(data)
self.url = "http://%s:%s/subscribe" % (dc._server_ip, dc._server_port)
if f:
# asynch - callback when new info is received
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = self.service_type,
status = ConnectionStatus.INIT,
message = 'Subscribe',
server_addrs = ['%s:%s' % (dc._server_ip, dc._server_port)])
self.task = gevent.spawn(self.ttl_loop)
else:
self._query()
self.done = True
def _resync_all_domains(self):
if not self._ks_domains_list:
# < keystonev3, no domains
return False
self._get_keystone_conn()
# compare new and old set,
# optimize for common case where nothing has changed,
# so track the project-ids in a set add '-',
# keystone gives uuid without...
try:
# The Default domain in ks(for v2 support) has id of 'default'
# replace with uuid of default-domain in vnc
ks_domain_ids = set([
str(uuid.UUID(dom['id'])) for dom in self._ks_domains_list()
if dom['id'] != 'default'
])
ks_domain_ids.add(self._vnc_default_domain_id)
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone',
status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %
(e, datetime.utcnow()),
server_addrs=[self._auth_url])
return True # retry
vnc_domain_ids = self._vnc_domain_ids
if vnc_domain_ids == ks_domain_ids:
# no change, go back to poll
return False
for vnc_domain_id in vnc_domain_ids - ks_domain_ids:
self.q.put((Q_DELETE, 'domain', vnc_domain_id))
if self._keystone_sync_on_demand:
# pre_domain_read will get it
pass
else:
for ks_domain_id in ks_domain_ids - vnc_domain_ids:
self.q.put((Q_CREATE, 'domain', ks_domain_id))
self.q.join()
gevent.sleep(0)
# we are in sync
self._vnc_domain_ids = ks_domain_ids
return False
def _resync_all_domains(self):
if not self._ks_domains_list:
# < keystonev3, no domains
return False
self._get_keystone_conn()
# compare new and old set,
# optimize for common case where nothing has changed,
# so track the project-ids in a set add '-',
# keystone gives uuid without...
try:
# The Default domain in ks(for v2 support) has id of 'default'
# replace with uuid of default-domain in vnc
ks_domain_ids = set(
[str(uuid.UUID(dom['id']))
for dom in self._ks_domains_list() if dom['id'] != 'default'])
ks_domain_ids.add(self._vnc_default_domain_id)
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
return True # retry
vnc_domain_ids = self._vnc_domain_ids
if vnc_domain_ids == ks_domain_ids:
# no change, go back to poll
return False
for vnc_domain_id in vnc_domain_ids - ks_domain_ids:
self.q.put((Q_DELETE, 'domain', vnc_domain_id))
if self._keystone_sync_on_demand:
# pre_domain_read will get it
pass
else:
for ks_domain_id in ks_domain_ids - vnc_domain_ids:
self.q.put((Q_CREATE, 'domain', ks_domain_id))
self.q.join()
gevent.sleep(0)
# we are in sync
self._vnc_domain_ids = ks_domain_ids
return False
def _ksv2_project_get(self, id):
# Note: under certain circumstances (if it has been initailized
# before endpoints are populated in keystone) keystoneclient may
# be valid to list projects, but not to read them. As it won't
# be reset by resync_all_projects, it is reseted on error here.
try:
return {'name': self._ks.tenants.get(id).name}
except Exception as e:
if self._ks is not None:
self._ks = None
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.DOWN,
message='Error: %s at UTC %s' %(e, datetime.utcnow()),
server_addrs=[self._auth_url])
self._get_keystone_conn()
return {'name': self._ks.tenants.get(id).name}
def _init_conn(self):
self._mapclient = client(("%s" % (self._ifmap_srv_ip),
"%s" % (self._ifmap_srv_port)),
self._username, self._password,
self._NAMESPACES, self._ssl_options)
connected = False
while not connected:
try:
resp_xml = self._mapclient.call('newSession', NewSessionRequest())
except socket.error as e:
msg = 'Failed to establish IF-MAP connection: %s' % str(e)
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
continue
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if not err_codes:
connected = True
else:
msg = "Failed to establish IF-MAP connection: %s" % err_codes
self.config_log(msg, level=SandeshLevel.SYS_WARN)
session_id = self._mapclient.get_session_id()
try:
self._mapclient.call('endSession',
EndSessionRequest(session_id))
except socket.error as e:
msg = "Failed to end the IF-MAP session %s: %s" %\
(session_id, str(e))
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
ConnectionState.update(conn_type = ConnectionType.IFMAP,
name = 'IfMap', status = ConnectionStatus.UP, message = '',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
self._conn_state = ConnectionStatus.UP
msg = 'IFMAP connection ESTABLISHED'
self.config_log(msg, level=SandeshLevel.SYS_NOTICE)
self._mapclient.set_session_id(
newSessionResult(resp_xml).get_session_id())
self._mapclient.set_publisher_id(
newSessionResult(resp_xml).get_publisher_id())
def _health_checker(self):
while True:
try:
# do the healthcheck only if we are connected
if self._conn_state == ConnectionStatus.DOWN:
continue
meta = Metadata('display-name', '',
{'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail', elements='')
request_str = self._build_request('healthcheck', 'self', [meta])
self._publish_to_ifmap_enqueue('update', request_str, do_trace=False)
# Confirm the existence of the following default global entities in IFMAP.
search_list = [
('global-system-config', ['default-global-system-config']),
]
for type, fq_name in search_list:
if not entity_is_present(self._mapclient, type, fq_name):
raise Exception("%s not found in IFMAP DB" % ':'.join(fq_name))
# If we had unpublished the IFMAP server to discovery server earlier
# publish it back now since it is valid now.
if not self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('up', '')
self._is_ifmap_up = True
ConnectionState.update(conn_type = ConnectionType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.UP,
message = '',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
except Exception as e:
log_str = 'IFMAP Healthcheck failed: %s' %(str(e))
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
if self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('down',
'IFMAP DB - Invalid state')
self._is_ifmap_up = False
ConnectionState.update(conn_type = ConnectionType.IFMAP,
name = 'IfMap',
status = ConnectionStatus.DOWN,
message = 'Invalid IFMAP DB State',
server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
self._ifmap_srv_port)])
finally:
gevent.sleep(
self._get_api_server().get_ifmap_health_check_interval())
def _sandesh_connection_info_update(self, status, message):
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER,
name = 'Zookeeper', status = new_conn_state,
message = message,
server_addrs = self._zookeeper_srvr.split(','))
if ((self._conn_state and self._conn_state != ConnectionStatus.DOWN) and
new_conn_state == ConnectionStatus.DOWN):
msg = 'Connection to Zookeeper down: %s' %(message)
self._supress_log(msg)
if (self._conn_state and self._conn_state != new_conn_state and
new_conn_state == ConnectionStatus.UP):
msg = 'Connection to Zookeeper ESTABLISHED'
self._supress_log(msg)
self._conn_state = new_conn_state
def _get_keystone_conn(self):
if self._ks:
return
if 'v3' in self._auth_url.split('/')[-1]:
self._ks = self._ksv3_get_conn()
if self._endpoint_type and self._ks.service_catalog:
self._ks.management_url = \
self._ks.service_catalog.get_urls(
service_type='identity',
endpoint_type=self._endpoint_type)[0]
else:
self._ks = self._ksv2_get_conn()
ConnectionState.update(conn_type=ConnType.OTHER,
name='Keystone', status=ConnectionStatus.UP, message='',
server_addrs=[self._auth_url])
def __init__(self, redis_uve_list, logger,
redis_password=None, \
uvedbcache=None, usecache=False, freq=5):
self._logger = logger
self._redis = None
self._uvedbcache = uvedbcache
self._usecache = usecache
self._redis_password = redis_password
self._uve_reverse_map = {}
self._freq = freq
for h,m in UVE_MAP.iteritems():
self._uve_reverse_map[m] = h
# Fill in redis/collector instances
self._redis_uve_map = {}
for new_elem in redis_uve_list:
test_elem = RedisInstKey(ip=new_elem[0], port=new_elem[1])
self._redis_uve_map[test_elem] = RedisInst()
ConnectionState.update(ConnectionType.REDIS_UVE,\
test_elem.ip+":"+str(test_elem.port), ConnectionStatus.INIT)
def update_redis_uve_list(self, redis_uve_list):
newlist = set(redis_uve_list)
# if some redis instances are gone, remove them from our map
for test_elem in self._redis_uve_map.keys():
r_ip = test_elem[0]
r_port = test_elem[1]
redis_inst = (r_ip, int(r_port))
if redis_inst not in newlist:
self._redis_uve_map[test_elem].deleted = True
else:
self._redis_uve_map[test_elem].deleted = False
# new redis instances need to be inserted into the map
for new_elem in newlist:
new_redis = RedisInstKey(ip=new_elem[0], port=new_elem[1])
if new_redis not in self._redis_uve_map:
self._redis_uve_map[new_redis] = RedisInst()
ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,\
name = new_elem[0]+":"+str(new_elem[1]), status = \
ConnectionStatus.INIT, server_addrs = \
[new_elem[0]+":"+str(new_elem[1])])
def _sandesh_connection_info_update(self, status, message):
from pysandesh.connection_info import ConnectionState
from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, ConnectionType
new_conn_state = getattr(ConnectionStatus, status)
ConnectionState.update(
conn_type=ConnectionType.ZOOKEEPER,
name="Zookeeper",
status=new_conn_state,
message=message,
server_addrs=self._zookeeper_srvr.split(","),
)
if (self._conn_state and self._conn_state != ConnectionStatus.DOWN) and new_conn_state == ConnectionStatus.DOWN:
msg = "Connection to Zookeeper down: %s" % (message)
self._supress_log(msg)
if self._conn_state and self._conn_state != new_conn_state and new_conn_state == ConnectionStatus.UP:
msg = "Connection to Zookeeper ESTABLISHED"
self._supress_log(msg)
self._conn_state = new_conn_state
def run(self):
ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,
name = 'LOCAL', status = ConnectionStatus.INIT)
while True:
if self._redis:
redish = self._redis
else:
redish = redis.StrictRedis(self._local_redis_uve[0],
self._local_redis_uve[1],
password=self._redis_password,
db=1)
try:
if not self._redis:
value = ""
redish.ping()
else:
k, value = redish.brpop("DELETED")
self._logger.debug("%s del received for " % value)
# value is of the format:
# DEL:<key>:<src>:<node-type>:<module>:<instance-id>:<message-type>:<seqno>
redish.delete(value)
except:
if self._redis:
#send redis connection down msg. Coule be bcos of authentication
ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,
name = 'LOCAL', status = ConnectionStatus.DOWN)
self._redis = None
gevent.sleep(5)
else:
self._logger.debug("Deleted %s" % value)
if not self._redis:
self._redis = redish
ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,
name = 'LOCAL', status = ConnectionStatus.UP)
def _publish_int(self, service, data, oper_state = 'up', msg = ''):
self.syslog('Publish service "%s", data "%s"' % (service, data))
payload = {
service : data,
'service-type' : service,
'remote-addr' : self.remote_addr,
'oper-state' : oper_state,
'oper-state-reason' : msg
}
emsg = None
cookie = None
try:
self.inc_pub_stats(service, 'request')
r = requests.post(
self.puburl, data=json.dumps(payload), headers=self._headers, timeout=30)
if r.status_code != 200:
self.inc_pub_stats(service, 'sc_%d' % r.status_code)
emsg = 'Status Code ' + str(r.status_code)
except requests.exceptions.ConnectionError:
self.inc_pub_stats(service, 'conn_error')
emsg = 'Connection Error'
except requests.exceptions.Timeout:
self.inc_pub_stats(service, 'timeout')
emsg = 'Request Timeout'
finally:
ConnectionState.update(conn_type = ConnectionType.DISCOVERY,
name = service,
status = ConnectionStatus.DOWN if emsg else ConnectionStatus.UP,
server_addrs = ['%s:%s' % (self._server_ip, \
self._server_port)],
message = 'Publish Error - %s' % emsg if emsg else 'Publish Success')
if not emsg:
self.inc_pub_stats(service, 'response')
self.inc_pub_stats(service, 'blob', value = json.dumps(data))
response = r.json()
cookie = response['cookie']
self.pubdata[cookie] = (service, data)
self.syslog('Saving token %s' % (cookie))
return cookie