def _cassandra_init_conn_pools(self): for ks, cf_list in self._keyspaces.items(): pool = pycassa.ConnectionPool( ks, self._server_list, max_overflow=-1, use_threadlocal=True, prefill=True, pool_size=20, pool_timeout=120, max_retries=-1, timeout=5, ) rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM for (cf, _) in cf_list: self._cf_dict[cf] = ColumnFamily( pool, cf, read_consistency_level=rd_consistency, write_consistency_level=wr_consistency ) ConnectionState.update( conn_type=ConnectionType.DATABASE, name="Cassandra", status=ConnectionStatus.UP, message="", server_addrs=self._server_list, ) self._conn_state = ConnectionStatus.UP msg = "Cassandra connection ESTABLISHED" self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _ksv3_project_get(self, id=None, name=None): if id: try: project = self._ks.projects.get(id) return { 'id': project.id, 'name': project.name, 'domain_id': project.domain_id } except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' % (e, datetime.utcnow()), server_addrs=[self._auth_url]) self._get_keystone_conn() project = self._ks.projects.get(id) return { 'id': project.id, 'name': project.name, 'domain_id': project.domain_id } else: id = None for tenant in self._ks.projects.list(): if tenant.name == name: id = tenant.id break return {'name': name, 'id': id}
def _cassandra_init_conn_pools(self): socket_factory = self._make_socket_factory() for ks, cf_dict in itertools.chain(list(self._rw_keyspaces.items()), list(self._ro_keyspaces.items())): keyspace = '%s%s' % (self._db_prefix, ks) pool = pycassa.ConnectionPool(keyspace, self._server_list, max_overflow=5, use_threadlocal=True, prefill=True, pool_size=self._pool_size, pool_timeout=120, max_retries=15, timeout=5, credentials=self._credential, socket_factory=socket_factory) for cf_name in cf_dict: cf_kwargs = cf_dict[cf_name].get('cf_args', {}) self._cf_dict[cf_name] = ColumnFamily( pool, cf_name, read_consistency_level=ConsistencyLevel.QUORUM, write_consistency_level=ConsistencyLevel.QUORUM, dict_class=dict, **cf_kwargs) ConnectionState.update(conn_type=ConnType.DATABASE, name='Cassandra', status=ConnectionStatus.UP, message='', server_addrs=self._server_list) self._conn_state = ConnectionStatus.UP msg = 'Cassandra connection ESTABLISHED' self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _redis_inst_up(self, r_inst, redish): if not self._redis_uve_map[r_inst]: r_ip = r_inst[0] r_port = r_inst[1] self._redis_uve_map[r_inst] = redish ConnectionState.update(ConnectionType.REDIS_UVE, r_ip + ":" + str(r_port), ConnectionStatus.UP)
def _publish_to_ifmap(self, oper_body): try: not_published = True retry_count = 0 resp_xml = None while not_published: sess_id = self._mapclient.get_session_id() req_xml = PublishRequest(sess_id, oper_body) resp_xml = self._mapclient.call('publish', req_xml) resp_doc = etree.parse(StringIO.StringIO(resp_xml)) err_codes = resp_doc.xpath( '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode', namespaces=self._NAMESPACES) if err_codes: if retry_count == 0: log_str = 'Error publishing to ifmap, req: %s, resp: %s' \ %(req_xml, resp_xml) self.config_log(log_str, level=SandeshLevel.SYS_ERR) ConnectionState.update( conn_type = ConnType.IFMAP, name = 'IfMap', status = ConnectionStatus.INIT, message = 'Session lost, renew it', server_addrs = ["%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port)]) self._conn_state = ConnectionStatus.INIT self._is_ifmap_up = False retry_count = retry_count + 1 self._init_conn() if self._ifmap_restarted(): msg = "IF-MAP servers restarted, re-populate it" self.config_log(msg, level=SandeshLevel.SYS_ERR) self.reset() self._get_api_server().publish_ifmap_to_discovery( 'down', msg) self._publish_config_root() self._db_client_mgr.db_resync() self._publish_to_ifmap_enqueue('publish_discovery', 1) else: # successful publish not_published = False break # end while not_published if retry_count: log_str = 'Success publishing to ifmap after %d tries' \ %(retry_count) self.config_log(log_str, level=SandeshLevel.SYS_ERR) return True, resp_xml except Exception as e: # Failed to publish the operation due to unknown error. # Probably a connection issue with the ifmap server. msg = "Failed to publish request %s: %s" % (oper_body, str(e)) return False, msg
def _cassandra_init_conn_pools(self): for ks, cf_list in self._keyspaces.items(): pool = pycassa.ConnectionPool(ks, self._server_list, max_overflow=-1, use_threadlocal=True, prefill=True, pool_size=20, pool_timeout=120, max_retries=-1, timeout=5) rd_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM wr_consistency = pycassa.cassandra.ttypes.ConsistencyLevel.QUORUM for (cf, _) in cf_list: self._cf_dict[cf] = ColumnFamily( pool, cf, read_consistency_level=rd_consistency, write_consistency_level=wr_consistency) ConnectionState.update(conn_type=ConnectionType.DATABASE, name='Cassandra', status=ConnectionStatus.UP, message='', server_addrs=self._server_list) self._conn_state = ConnectionStatus.UP msg = 'Cassandra connection ESTABLISHED' self._logger(msg, level=SandeshLevel.SYS_NOTICE)
def _query(self): conn_state_updated = False # hoping all errors are transient and a little wait will solve the problem while True: try: self.stats['request'] += 1 r = requests.post(self.url, data=self.post_body, headers=self._headers, timeout=5) if r.status_code == 200: break self.inc_stats('sc_%d' % r.status_code) emsg = "Status Code %d" % r.status_code except requests.exceptions.ConnectionError: self.stats['conn_error'] += 1 emsg = 'Connection Error' except (requests.exceptions.Timeout, socket.timeout): self.stats['timeout'] += 1 emsg = 'Request Timeout' self.syslog('connection error or failed to subscribe') if not conn_state_updated: conn_state_updated = True ConnectionState.update( conn_type = ConnectionType.DISCOVERY, name = self.service_type, status = ConnectionStatus.DOWN, message = 'Subscribe - %s' % emsg, server_addrs = \ ['%s:%s' % (self.dc._server_ip, \ self.dc._server_port)]) gevent.sleep(2) # end while self.syslog('query resp => %s ' % r.text) response = r.json() # avoid signature on ttl which can change between iterations info = response[self.service_type] infostr = json.dumps(info) sig = hashlib.md5(infostr).hexdigest() self.stats['response'] += 1 self.stats['ttl'] = response['ttl'] self.stats['blob'] = infostr self.ttl = response['ttl'] self.change = False if sig != self.sig: #print 'signature mismatch! old=%s, new=%s' % (self.sig, sig) self.info = info self.sig = sig self.change = True ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = self.service_type, status = ConnectionStatus.UP, message = 'Subscribe Response', server_addrs = ['%s:%s' % (self.dc._server_ip, \ self.dc._server_port)])
def _ksv2_project_get(self, id=None, name=None): # Note: under certain circumstances (if it has been initailized # before endpoints are populated in keystone) keystoneclient may # be valid to list projects, but not to read them. As it won't # be reset by resync_all_projects, it is reseted on error here. if id: try: return {'name': self._ks.tenants.get(id).name, 'id': id} except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' % (e, datetime.utcnow()), server_addrs=[self._auth_url]) self._get_keystone_conn() return {'name': self._ks.tenants.get(id).name, 'id': id} else: id = None for tenant in self._ks.tenants.list(): if tenant.name == name: id = tenant.id break return {'name': name, 'id': id}
def __init__(self, redis_uve_list, logger, redis_password=None, redis_ssl_params=None, \ uvedbcache=None, usecache=False, freq=5): self._logger = logger self._redis = None self._uvedbcache = uvedbcache self._usecache = usecache self._redis_cfg_info = [] self._redis_password = redis_password self._redis_ssl_params = redis_ssl_params self._uve_reverse_map = {} self._freq = freq self._active_collectors = [] for h, m in UVE_MAP.items(): self._uve_reverse_map[m] = h # Fill in redis/collector instances self._redis_uve_map = {} for new_elem in redis_uve_list: test_elem = RedisInstKey(ip=new_elem[0], port=new_elem[1]) self._redis_cfg_info.append(test_elem) self._redis_uve_map[test_elem] = RedisInst() ConnectionState.update(ConnectionType.REDIS_UVE,\ test_elem.ip+":"+str(test_elem.port), ConnectionStatus.INIT, [test_elem.ip+":"+str(test_elem.port)],"Redis Instance initialized")
def update_redis_uve_list(self, redis_uve_list): newlist = set(redis_uve_list) chg = False # if some redis instances are gone, remove them from our map for test_elem in self._redis_uve_map.keys(): if test_elem not in newlist: chg = True r_ip = test_elem[0] r_port = test_elem[1] del self._redis_uve_map[test_elem] ConnectionState.delete(ConnectionType.REDIS_UVE,\ r_ip+":"+str(r_port)) # new redis instances need to be inserted into the map for test_elem in newlist: if test_elem not in self._redis_uve_map: chg = True r_ip = test_elem[0] r_port = test_elem[1] self._redis_uve_map[test_elem] = None ConnectionState.update(ConnectionType.REDIS_UVE,\ r_ip+":"+str(r_port), ConnectionStatus.INIT) if chg: self._logger.error("updated redis_uve_list %s" % str(self._redis_uve_map)) # Exercise redis connections to update health if len(newlist): self.get_uve("ObjectCollectorInfo:__NONE__", False, None)
def heartbeat(self): while True: # send heartbeat for each published object seperately # dictionary can change size during iteration pub_list = self.pubdata.copy() for cookie in pub_list: payload = {'cookie': cookie} self.syslog('Sending cookie %s in heartbeat' % cookie) try: r = requests.post( self.hburl, data=json.dumps(payload), headers=self._headers) except requests.exceptions.ConnectionError: service, data = pub_list[cookie] ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = service, status = ConnectionStatus.DOWN, server_addrs = ['%s:%s' % (self._server_ip, \ self._server_port)], message = 'HeartBeat - Connection Error') self.syslog('Connection Error') continue # if DS lost track of our data, republish if r.status_code == 404: # forget cached cookie and object; will re-learn self.syslog('Server lost track of token %s' % (cookie)) service, data = self.pubdata[cookie] del self.pubdata[cookie] self._publish_int(service, data) gevent.sleep(HC_INTERVAL)
def conn_state_notify(self, svc, msg='', up=True, servers=''): ctype = self.map_svc(svc) status = self.map_sts(up) if isinstance(servers, basestring): servers = [servers] ConnectionState.update(conn_type=ctype, name='SNMP', status=status, message=msg, server_addrs=servers)
def _redis_inst_down(self, r_inst): if self._redis_uve_map[r_inst]: r_ip = r_inst[0] r_port = r_inst[1] self._redis_uve_map[r_inst] = None ConnectionState.update(ConnectionType.REDIS_UVE, r_ip + ":" + str(r_port), ConnectionStatus.DOWN)
def connection_state_update(self, status, message=None): ConnectionState.update( conn_type=ConnType.APISERVER, name='ApiServer', status=status, message=message or 'ApiServer Connection State updated', server_addrs=['%s:%s' % (self._args.api_server_ip, self._args.api_server_port)])
def heartbeat(self): while True: # send heartbeat for each published object seperately # dictionary can change size during iteration pub_list = self.pubdata.copy() for cookie in pub_list: payload = {'cookie': cookie} self.syslog('Sending cookie %s in heartbeat' % cookie) try: r = requests.post( self.hburl, data=json.dumps(payload), headers=self._headers) except requests.exceptions.ConnectionError: service, data = pub_list[cookie] ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = service, status = ConnectionState.DOWN, server_addrs = ['%s:%s' % (self._server_ip, \ self._server_port)], message = 'HeartBeat - Connection Error') self.syslog('Connection Error') continue # if DS lost track of our data, republish if r.status_code == 404: # forget cached cookie and object; will re-learn self.syslog('Server lost track of token %s' % (cookie)) service, data = self.pubdata[cookie] del self.pubdata[cookie] self._publish_int(service, data) gevent.sleep(HC_INTERVAL)
def vnc_api(self): e = SystemError('Cant connect to API server') api_servers = [srv.split(':')[0] for srv in self._args.api_server_list] api_server_port = self._args.api_server_list[0].split(':')[1] \ if self._args.api_server_list else None try: vnc = VncApi(self._args.admin_user, self._args.admin_password, self._args.admin_tenant_name, api_servers, api_server_port, api_server_use_ssl=self._args.api_server_use_ssl, auth_host=self._args.auth_host, auth_port=self._args.auth_port, auth_protocol=self._args.auth_protocol) except Exception as e: ConnectionState.update(conn_type=ConnectionType.APISERVER, name='Config', status=ConnectionStatus.DOWN, message=str(e), server_addrs=api_servers) return None else: ConnectionState.update(conn_type=ConnectionType.APISERVER, name='Config', status=ConnectionStatus.UP, server_addrs=api_servers) return vnc
def _update_sandesh_status(self, status, msg=''): ConnectionState.update( conn_type=ConnectionType.DATABASE, name='RabbitMQ', status=status, message=msg, server_addrs=["%s:%s" % (self._rabbit_ip, self._rabbit_port)])
def _publish_to_ifmap(self, oper_body): try: not_published = True retry_count = 0 resp_xml = None while not_published: sess_id = self._mapclient.get_session_id() req_xml = PublishRequest(sess_id, oper_body) resp_xml = self._mapclient.call('publish', req_xml) resp_doc = etree.parse(StringIO.StringIO(resp_xml)) err_codes = resp_doc.xpath( '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode', namespaces=self._NAMESPACES) if err_codes: if retry_count == 0: log_str = 'Error publishing to ifmap, req: %s, resp: %s' \ %(req_xml, resp_xml) self.config_log(log_str, level=SandeshLevel.SYS_ERR) ConnectionState.update( conn_type = ConnectionType.IFMAP, name = 'IfMap', status = ConnectionStatus.INIT, message = 'Session lost, renew it', server_addrs = ["%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port)]) self._conn_state = ConnectionStatus.INIT self._is_ifmap_up = False retry_count = retry_count + 1 self._init_conn() if self._ifmap_restarted(): msg = "IF-MAP servers restarted, re-populate it" self.config_log(msg, level=SandeshLevel.SYS_ERR) self.reset() self._get_api_server().publish_ifmap_to_discovery( 'down', msg) self._publish_config_root() self._db_client_mgr.db_resync() self._publish_to_ifmap_enqueue('publish_discovery', 1) else: # successful publish not_published = False break # end while not_published if retry_count: log_str = 'Success publishing to ifmap after %d tries' \ %(retry_count) self.config_log(log_str, level=SandeshLevel.SYS_ERR) return True, resp_xml except Exception as e: # Failed to publish the operation due to unknown error. # Probably a connection issue with the ifmap server. msg = "Failed to publish request %s: %s" % (oper_body, str(e)) return False, msg
def run(self): lck = False while True: try: k, value = self._redis.brpop("DELETED") self._sem.acquire() lck = True self._logger.debug("%s del received for " % value) # value is of the format: # DEL:<key>:<src>:<node-type>:<module>:<instance-id>:<message-type>:<seqno> self._redis.delete(value) except redis.exceptions.ResponseError: #send redis connection down msg. Coule be bcos of authentication ConnectionState.update(conn_type = ConnectionType.REDIS, name = 'UVE', status = ConnectionStatus.DOWN, message = 'UVE result : Connection Error', server_addrs = ['%s:%d' % (self._local_redis_uve[0], self._local_redis_uve[1])]) sys.exit() except redis.exceptions.ConnectionError: if lck: self._sem.release() lck = False gevent.sleep(5) else: if lck: self._sem.release() lck = False self._logger.debug("Deleted %s" % value) self._logger.debug("UVE %s Type %s" % (key, typ))
def run(self): exitrun = False while not exitrun: for rkey in self._redis_uve_map.keys(): rinst = self._redis_uve_map[rkey] old_pid = rinst.collector_pid try: # check if it is marked as deleted during sighup handling if rinst.deleted == True: r_ip = rkey[0] r_port = rkey[1] del self._redis_uve_map[rkey] ConnectionState.delete(ConnectionType.REDIS_UVE,\ r_ip+":"+str(r_port)) continue if rinst.redis_handle is None: rinst.redis_handle = StrictRedisWrapper( host=rkey.ip, port=rkey.port, password=self._redis_password, db=1, socket_timeout=30) rinst.collector_pid = None # check for known collector pid string # if there's a mismatch, we must read it again if rinst.collector_pid is not None: if not rinst.redis_handle.sismember( "NGENERATORS", rinst.collector_pid): rinst.collector_pid = None # read the collector pid string if rinst.collector_pid is None: for gen in rinst.redis_handle.smembers("NGENERATORS"): module = gen.split(':')[2] if module == "contrail-collector": rinst.collector_pid = gen except gevent.GreenletExit: self._logger.error('UVEServer Exiting on gevent-kill') exitrun = True break except Exception as e: self._logger.error("redis/collector healthcheck failed %s for %s" \ % (str(e), str(rkey))) rinst.redis_handle = None rinst.collector_pid = None finally: # Update redis/collector health if old_pid is None and rinst.collector_pid is not None: ConnectionState.update(ConnectionType.REDIS_UVE,\ rkey.ip + ":" + str(rkey.port), ConnectionStatus.UP, [rkey.ip+":"+str(rkey.port)]) if old_pid is not None and rinst.collector_pid is None: ConnectionState.update(ConnectionType.REDIS_UVE,\ rkey.ip + ":" + str(rkey.port), ConnectionStatus.DOWN, [rkey.ip+":"+str(rkey.port)]) if not exitrun: gevent.sleep(self._freq)
def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh): global openstack_driver openstack_driver = self if api_server_ip == '0.0.0.0': self._vnc_api_ip = '127.0.0.1' else: self._vnc_api_ip = api_server_ip self._vnc_api_port = api_server_port self._config_sections = conf_sections fill_keystone_opts(self, conf_sections) if 'v3' in self._auth_url.split('/')[-1]: self._ks_domains_list = self._ksv3_domains_list self._ks_domain_get = self._ksv3_domain_get self._ks_projects_list = self._ksv3_projects_list self._ks_project_get = self._ksv3_project_get self.sync_project_to_vnc = self._ksv3_sync_project_to_vnc self._add_project_to_vnc = self._ksv3_add_project_to_vnc self._del_project_from_vnc = self._ksv3_del_project_from_vnc self._vnc_default_domain_id = None else: self._ks_domains_list = None self._ks_domain_get = None self._ks_projects_list = self._ksv2_projects_list self._ks_project_get = self._ksv2_project_get self.sync_project_to_vnc = self._ksv2_sync_project_to_vnc self._add_project_to_vnc = self._ksv2_add_project_to_vnc self._del_project_from_vnc = self._ksv2_del_project_from_vnc self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.INIT, message='', server_addrs=[self._auth_url]) self._vnc_lib = None # resync failures, don't retry forever self._failed_domain_dels = set() self._failed_project_dels = set() # active domains/projects in contrail/vnc api server self._vnc_domain_ids = set() self._vnc_project_ids = set() # logging self._sandesh_logger = sandesh.logger() self._vnc_os_logger = logging.getLogger(__name__) self._vnc_os_logger.setLevel(logging.ERROR) # Add the log message handler to the logger try: with open(self._err_file, 'a'): handler = logging.handlers.RotatingFileHandler( self._err_file, maxBytes=64*1024, backupCount=5) self._vnc_os_logger.addHandler(handler) except IOError: self._sandesh_logger.error("Failed to open trace file %s" % self._err_file) self.q = Queue.Queue(maxsize=Q_MAX_ITEMS)
def _redis_inst_up(self, r_inst, redish): if r_inst in self._redis_uve_map and not self._redis_uve_map[r_inst]: r_ip = r_inst[0] r_port = r_inst[1] self._redis_uve_map[r_inst] = redish ConnectionState.update(ConnectionType.REDIS_UVE,\ r_ip + ":" + str(r_port), ConnectionStatus.UP, [r_ip + ":" + str(r_port)], message = 'Redis Instance is up')
def _update_sandesh_status(self, status, msg=''): ConnectionState.update(conn_type=ConnType.DATABASE, name='Cassandra', status=status, message=msg, server_addrs=self._server_list) # Keeps trace of the current status. self._conn_state = status
def _update_sandesh_status(self, status, msg=""): ConnectionState.update( conn_type=ConnectionType.DATABASE, name="Cassandra", status=status, message=msg, server_addrs=self._server_list, )
def _update_sandesh_status(self, status, msg=""): ConnectionState.update( conn_type=ConnectionType.DATABASE, name="RabbitMQ", status=status, message=msg, server_addrs=["%s:%s" % (self._rabbit_ip, self._rabbit_port)], )
def _update_connection_state(self, status, message=''): server_addrs = ['%s:%d' % (self._conf_info['api_server_ip'], \ self._conf_info['api_server_port'])] ConnectionState.update(conn_type=ConnectionType.APISERVER, name='', status=status, message=message, server_addrs=server_addrs)
def connection_state_update(self, status, message=None): ConnectionState.update( conn_type=ConnectionType.APISERVER, name="ApiServer", status=status, message=message or "", server_addrs=["%s:%s" % (self._args.api_server_ip, self._args.api_server_port)], )
def _sandesh_connection_info_update(self, status, message): from pysandesh.connection_info import ConnectionState from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \ ConnectionType ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER, name = 'Zookeeper', status = getattr(ConnectionStatus, status), message = message, server_addrs = self._server_list.split(','))
def _health_checker(self): while True: try: # do the healthcheck only if we are connected if self._conn_state == ConnectionStatus.DOWN: continue meta = Metadata('display-name', '', {'ifmap-cardinality': 'singleValue'}, ns_prefix='contrail', elements='') request_str = self._build_request('healthcheck', 'self', [meta]) self._publish_to_ifmap_enqueue('update', request_str, do_trace=False) # Confirm the existence of the following default global entities in IFMAP. search_list = [ ('global-system-config', ['default-global-system-config']), ] for type, fq_name in search_list: if not entity_is_present(self._mapclient, type, fq_name): raise Exception("%s not found in IFMAP DB" % ':'.join(fq_name)) # If we had unpublished the IFMAP server to discovery server earlier # publish it back now since it is valid now. if not self._is_ifmap_up: self._get_api_server().publish_ifmap_to_discovery('up', '') self._is_ifmap_up = True ConnectionState.update( conn_type=ConnectionType.IFMAP, name='IfMap', status=ConnectionStatus.UP, message='', server_addrs=[ "%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port) ]) except Exception as e: log_str = 'IFMAP Healthcheck failed: %s' % (str(e)) self.config_log(log_str, level=SandeshLevel.SYS_ERR) if self._is_ifmap_up: self._get_api_server().publish_ifmap_to_discovery( 'down', 'IFMAP DB - Invalid state') self._is_ifmap_up = False ConnectionState.update( conn_type=ConnectionType.IFMAP, name='IfMap', status=ConnectionStatus.DOWN, message='Invalid IFMAP DB State', server_addrs=[ "%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port) ]) finally: gevent.sleep( self._get_api_server().get_ifmap_health_check_interval())
def _query(self): conn_state_updated = False # hoping all errors are transient and a little wait will solve the problem while True: try: self.stats['request'] += 1 r = requests.post( self.url, data=self.post_body, headers=self._headers, timeout=5) if r.status_code == 200: break self.inc_stats('sc_%d' % r.status_code) emsg = "Status Code %d" % r.status_code except requests.exceptions.ConnectionError: self.stats['conn_error'] += 1 emsg = 'Connection Error' except (requests.exceptions.Timeout, socket.timeout): self.stats['timeout'] += 1 emsg = 'Request Timeout' self.syslog('connection error or failed to subscribe') if not conn_state_updated: conn_state_updated = True ConnectionState.update( conn_type = ConnectionType.DISCOVERY, name = self.service_type, status = ConnectionStatus.DOWN, message = 'Subscribe - %s' % emsg, server_addrs = \ ['%s:%s' % (self.dc._server_ip, \ self.dc._server_port)]) gevent.sleep(2) # end while self.syslog('query resp => %s ' % r.text) response = r.json() # avoid signature on ttl which can change between iterations info = response[self.service_type] infostr = json.dumps(info) sig = hashlib.md5(infostr).hexdigest() self.stats['response'] += 1 self.stats['ttl'] = response['ttl'] self.stats['blob'] = infostr self.ttl = response['ttl'] self.change = False if sig != self.sig: #print 'signature mismatch! old=%s, new=%s' % (self.sig, sig) self.info = info self.sig = sig self.change = True ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = self.service_type, status = ConnectionStatus.UP, message = 'Subscribe Response', server_addrs = ['%s:%s' % (self.dc._server_ip, \ self.dc._server_port)])
def __init__(self, dc, service_type, count, f=None, *args, **kw): super(Subscribe, self).__init__() self.verify = kw.pop("verify") self.protocol = "http" if self.verify: self.protocol = "https" self.dc = dc self.f = f self.kw = kw self.args = args self.count = count self.service_type = service_type self._headers = { 'Content-type': 'application/json', } self.info = [] infostr = json.dumps(self.info) self.sig = hashlib.md5(infostr).hexdigest() self.done = False self.stats = { 'service_type': service_type, 'request': 0, 'response': 0, 'conn_error': 0, 'timeout': 0, 'exc_unknown': 0, 'exc_info': '', 'instances': count, 'ttl': 0, 'blob': '', } self.data = { 'service': service_type, 'instances': count, 'client-type': dc._client_type, 'remote-addr': dc.remote_addr, 'client': dc._myid } self.post_body = json.dumps(self.data) self.url = "%s://%s:%s/subscribe" % (self.protocol, dc._server_ip, dc._server_port) if f: # asynch - callback when new info is received ConnectionState.update( conn_type=ConnectionType.DISCOVERY, name=self.service_type, status=ConnectionStatus.INIT, message='Subscribe', server_addrs=['%s:%s' % (dc._server_ip, dc._server_port)]) self.task = gevent.spawn(self.ttl_loop) else: self._query() self.done = True
def _redis_inst_down(self, r_inst): if r_inst in self._redis_uve_map and self._redis_uve_map[r_inst]: r_ip = r_inst[0] r_port = r_inst[1] self._redis_uve_map[r_inst] = None ConnectionState.update(ConnectionType.REDIS_UVE, r_ip + ":" + str(r_port), ConnectionStatus.DOWN, message='Redis Instance is down')
def run(self): exitrun = False while not exitrun: for rkey in self._redis_uve_map.keys(): rinst = self._redis_uve_map[rkey] old_pid = rinst.collector_pid try: # check if it is marked as deleted during sighup handling if rinst.deleted == True: r_ip = rkey[0] r_port = rkey[1] del self._redis_uve_map[rkey] ConnectionState.delete(ConnectionType.REDIS_UVE,\ r_ip+":"+str(r_port)) continue if rinst.redis_handle is None: rinst.redis_handle = StrictRedisWrapper( host=rkey.ip, port=rkey.port, password=self._redis_password, db=1, socket_timeout=30) rinst.collector_pid = None # check for known collector pid string # if there's a mismatch, we must read it again if rinst.collector_pid is not None: if not rinst.redis_handle.sismember("NGENERATORS", rinst.collector_pid): rinst.collector_pid = None # read the collector pid string if rinst.collector_pid is None: for gen in rinst.redis_handle.smembers("NGENERATORS"): module = gen.split(':')[2] if module == "contrail-collector": rinst.collector_pid = gen except gevent.GreenletExit: self._logger.error('UVEServer Exiting on gevent-kill') exitrun = True break except Exception as e: self._logger.error("redis/collector healthcheck failed %s for %s" \ % (str(e), str(rkey))) rinst.redis_handle = None rinst.collector_pid = None finally: # Update redis/collector health if old_pid is None and rinst.collector_pid is not None: ConnectionState.update(ConnectionType.REDIS_UVE,\ rkey.ip + ":" + str(rkey.port), ConnectionStatus.UP, [rkey.ip+":"+str(rkey.port)]) if old_pid is not None and rinst.collector_pid is None: ConnectionState.update(ConnectionType.REDIS_UVE,\ rkey.ip + ":" + str(rkey.port), ConnectionStatus.DOWN, [rkey.ip+":"+str(rkey.port)]) if not exitrun: gevent.sleep(self._freq)
def _get_keystone_conn(self): if self._ks: return if self._use_certs: verify = self._kscertbundle else: if self._insecure: verify = False else: verify = True if self._admin_token: auth = kauth.token.Token(self._auth_url, token=self._admin_token) else: kwargs = { 'project_name': self._admin_tenant, 'username': self._auth_user, 'password': self._auth_passwd, 'user_domain_name': self._user_domain_name, } if self._user_domain_name: kwargs.update({ 'project_domain_name': self._project_domain_name, 'project_name': self._project_name, }) else: kwargs.update({ 'domain_id': self._domain_id, }) auth = kauth.password.Password(self._auth_url, **kwargs) sess = ksession.Session(auth=auth, verify=verify) try: self._ks = kclient.Client(session=sess, auth_url=self._auth_url) except kexceptions.DiscoveryFailure: # Probably a v2 Keytone API, remove v3 args and try again v3_args = ['user_domain_name', 'project_domain_name', 'domain_id'] for arg in v3_args: kwargs.pop(arg, None) auth = kauth.password.Password(self._auth_url, **kwargs) sess = ksession.Session(auth=auth, verify=verify) self._ks = kclient.Client(session=sess, auth_url=self._auth_url) if self._endpoint_type and auth.auth_ref.service_catalog: self._ks.management_url = \ auth.auth_ref.service_catalog.get_urls( service_type='identity', endpoint_type=self._endpoint_type)[0] ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.UP, message='', server_addrs=[self._auth_url])
def connection_state_update(self, status, message=None): ConnectionState.update( conn_type=ConnType.APISERVER, name='ApiServer', status=status, message=message or '', server_addrs=[ '%s:%s' % (self.args.vnc_endpoint_ip, self.args.vnc_endpoint_port) ])
def _get_keystone_conn(self): if self._ks: return if self._use_certs: verify=self._kscertbundle else: if self._insecure: verify=False else: verify=True if self._admin_token: auth = kauth.token.Token(self._auth_url, token=self._admin_token) else: kwargs = { 'project_name': self._admin_tenant, 'username': self._auth_user, 'password': self._auth_passwd, 'user_domain_name': self._user_domain_name, } if self._user_domain_name: kwargs.update({ 'project_domain_name': self._project_domain_name, 'project_name': self._project_name, }) else: kwargs.update({ 'domain_id': self._domain_id, }) auth = kauth.password.Password(self._auth_url, **kwargs) sess = ksession.Session(auth=auth, verify=verify) try: self._ks = kclient.Client(session=sess, auth_url=self._auth_url) except kexceptions.DiscoveryFailure: # Probably a v2 Keytone API, remove v3 args and try again v3_args = ['user_domain_name', 'project_domain_name', 'domain_id'] for arg in v3_args: kwargs.pop(arg, None) auth = kauth.password.Password(self._auth_url, **kwargs) sess = ksession.Session(auth=auth, verify=verify) self._ks = kclient.Client(session=sess, auth_url=self._auth_url) if self._endpoint_type and auth.auth_ref.service_catalog: self._ks.management_url = \ auth.auth_ref.service_catalog.get_urls( service_type='identity', endpoint_type=self._endpoint_type)[0] ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.UP, message='', server_addrs=[self._auth_url])
def connection_state_update(status, message=None): ConnectionState.update(conn_type=ConnType.APISERVER, name='ApiServer', status=status, message=message or '', server_addrs=[ '%s:%s' % (args.api_server_ip, args.api_server_port) ]) st_logger.notice( "############# UPDATE CONNECTION STATE: {}".format(status))
def _ksv3_domain_get(self, id=None): try: return {'name': self._ks.domains.get(id).name} except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' %(e, datetime.utcnow()), server_addrs=[self._auth_url]) self._get_keystone_conn() return {'name': self._ks.domains.get(id).name}
def _init_conn(self): self._mapclient = client( ("%s" % (self._ifmap_srv_ip), "%s" % (self._ifmap_srv_port)), self._username, self._password, self._NAMESPACES, self._ssl_options) connected = False while not connected: try: resp_xml = self._mapclient.call('newSession', NewSessionRequest()) except socket.error as e: msg = 'Failed to establish IF-MAP connection: %s' % str(e) self.config_log(msg, level=SandeshLevel.SYS_WARN) time.sleep(3) continue resp_doc = etree.parse(StringIO.StringIO(resp_xml)) err_codes = resp_doc.xpath( '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode', namespaces=self._NAMESPACES) if not err_codes: connected = True else: msg = "Failed to establish IF-MAP connection: %s" % err_codes self.config_log(msg, level=SandeshLevel.SYS_WARN) session_id = self._mapclient.get_session_id() try: self._mapclient.call('endSession', EndSessionRequest(session_id)) except socket.error as e: msg = "Failed to end the IF-MAP session %s: %s" %\ (session_id, str(e)) self.config_log(msg, level=SandeshLevel.SYS_WARN) time.sleep(3) ConnectionState.update(conn_type=ConnectionType.IFMAP, name='IfMap', status=ConnectionStatus.UP, message='', server_addrs=[ "%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port) ]) self._conn_state = ConnectionStatus.UP msg = 'IFMAP connection ESTABLISHED' self.config_log(msg, level=SandeshLevel.SYS_NOTICE) self._mapclient.set_session_id( newSessionResult(resp_xml).get_session_id()) self._mapclient.set_publisher_id( newSessionResult(resp_xml).get_publisher_id())
def __init__(self, dc, service_type, count, f=None, *args, **kw): self.dc = dc self.f = f self.kw = kw self.args = args self.count = count self.service_type = service_type self._headers = { 'Content-type': 'application/json', } self.info = [] infostr = json.dumps(self.info) self.sig = hashlib.md5(infostr).hexdigest() self.done = False self.stats = { 'service_type' : service_type, 'request' : 0, 'response' : 0, 'conn_error' : 0, 'timeout' : 0, 'exc_unknown' : 0, 'exc_info' : '', 'instances' : count, 'ttl' : 0, 'blob' : '', } data = { 'service': service_type, 'instances': count, 'client-type': dc._client_type, 'remote-addr': dc._myip, 'client': dc._myid } self.post_body = json.dumps(data) self.url = "http://%s:%s/subscribe" % (dc._server_ip, dc._server_port) if f: # asynch - callback when new info is received ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = self.service_type, status = ConnectionStatus.INIT, message = 'Subscribe', server_addrs = ['%s:%s' % (dc._server_ip, dc._server_port)]) self.task = gevent.spawn(self.ttl_loop) else: self._query() self.done = True
def _resync_all_domains(self): if not self._ks_domains_list: # < keystonev3, no domains return False self._get_keystone_conn() # compare new and old set, # optimize for common case where nothing has changed, # so track the project-ids in a set add '-', # keystone gives uuid without... try: # The Default domain in ks(for v2 support) has id of 'default' # replace with uuid of default-domain in vnc ks_domain_ids = set([ str(uuid.UUID(dom['id'])) for dom in self._ks_domains_list() if dom['id'] != 'default' ]) ks_domain_ids.add(self._vnc_default_domain_id) except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' % (e, datetime.utcnow()), server_addrs=[self._auth_url]) return True # retry vnc_domain_ids = self._vnc_domain_ids if vnc_domain_ids == ks_domain_ids: # no change, go back to poll return False for vnc_domain_id in vnc_domain_ids - ks_domain_ids: self.q.put((Q_DELETE, 'domain', vnc_domain_id)) if self._keystone_sync_on_demand: # pre_domain_read will get it pass else: for ks_domain_id in ks_domain_ids - vnc_domain_ids: self.q.put((Q_CREATE, 'domain', ks_domain_id)) self.q.join() gevent.sleep(0) # we are in sync self._vnc_domain_ids = ks_domain_ids return False
def _resync_all_domains(self): if not self._ks_domains_list: # < keystonev3, no domains return False self._get_keystone_conn() # compare new and old set, # optimize for common case where nothing has changed, # so track the project-ids in a set add '-', # keystone gives uuid without... try: # The Default domain in ks(for v2 support) has id of 'default' # replace with uuid of default-domain in vnc ks_domain_ids = set( [str(uuid.UUID(dom['id'])) for dom in self._ks_domains_list() if dom['id'] != 'default']) ks_domain_ids.add(self._vnc_default_domain_id) except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' %(e, datetime.utcnow()), server_addrs=[self._auth_url]) return True # retry vnc_domain_ids = self._vnc_domain_ids if vnc_domain_ids == ks_domain_ids: # no change, go back to poll return False for vnc_domain_id in vnc_domain_ids - ks_domain_ids: self.q.put((Q_DELETE, 'domain', vnc_domain_id)) if self._keystone_sync_on_demand: # pre_domain_read will get it pass else: for ks_domain_id in ks_domain_ids - vnc_domain_ids: self.q.put((Q_CREATE, 'domain', ks_domain_id)) self.q.join() gevent.sleep(0) # we are in sync self._vnc_domain_ids = ks_domain_ids return False
def _ksv2_project_get(self, id): # Note: under certain circumstances (if it has been initailized # before endpoints are populated in keystone) keystoneclient may # be valid to list projects, but not to read them. As it won't # be reset by resync_all_projects, it is reseted on error here. try: return {'name': self._ks.tenants.get(id).name} except Exception as e: if self._ks is not None: self._ks = None ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.DOWN, message='Error: %s at UTC %s' %(e, datetime.utcnow()), server_addrs=[self._auth_url]) self._get_keystone_conn() return {'name': self._ks.tenants.get(id).name}
def _init_conn(self): self._mapclient = client(("%s" % (self._ifmap_srv_ip), "%s" % (self._ifmap_srv_port)), self._username, self._password, self._NAMESPACES, self._ssl_options) connected = False while not connected: try: resp_xml = self._mapclient.call('newSession', NewSessionRequest()) except socket.error as e: msg = 'Failed to establish IF-MAP connection: %s' % str(e) self.config_log(msg, level=SandeshLevel.SYS_WARN) time.sleep(3) continue resp_doc = etree.parse(StringIO.StringIO(resp_xml)) err_codes = resp_doc.xpath( '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode', namespaces=self._NAMESPACES) if not err_codes: connected = True else: msg = "Failed to establish IF-MAP connection: %s" % err_codes self.config_log(msg, level=SandeshLevel.SYS_WARN) session_id = self._mapclient.get_session_id() try: self._mapclient.call('endSession', EndSessionRequest(session_id)) except socket.error as e: msg = "Failed to end the IF-MAP session %s: %s" %\ (session_id, str(e)) self.config_log(msg, level=SandeshLevel.SYS_WARN) time.sleep(3) ConnectionState.update(conn_type = ConnectionType.IFMAP, name = 'IfMap', status = ConnectionStatus.UP, message = '', server_addrs = ["%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port)]) self._conn_state = ConnectionStatus.UP msg = 'IFMAP connection ESTABLISHED' self.config_log(msg, level=SandeshLevel.SYS_NOTICE) self._mapclient.set_session_id( newSessionResult(resp_xml).get_session_id()) self._mapclient.set_publisher_id( newSessionResult(resp_xml).get_publisher_id())
def _health_checker(self): while True: try: # do the healthcheck only if we are connected if self._conn_state == ConnectionStatus.DOWN: continue meta = Metadata('display-name', '', {'ifmap-cardinality': 'singleValue'}, ns_prefix='contrail', elements='') request_str = self._build_request('healthcheck', 'self', [meta]) self._publish_to_ifmap_enqueue('update', request_str, do_trace=False) # Confirm the existence of the following default global entities in IFMAP. search_list = [ ('global-system-config', ['default-global-system-config']), ] for type, fq_name in search_list: if not entity_is_present(self._mapclient, type, fq_name): raise Exception("%s not found in IFMAP DB" % ':'.join(fq_name)) # If we had unpublished the IFMAP server to discovery server earlier # publish it back now since it is valid now. if not self._is_ifmap_up: self._get_api_server().publish_ifmap_to_discovery('up', '') self._is_ifmap_up = True ConnectionState.update(conn_type = ConnectionType.IFMAP, name = 'IfMap', status = ConnectionStatus.UP, message = '', server_addrs = ["%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port)]) except Exception as e: log_str = 'IFMAP Healthcheck failed: %s' %(str(e)) self.config_log(log_str, level=SandeshLevel.SYS_ERR) if self._is_ifmap_up: self._get_api_server().publish_ifmap_to_discovery('down', 'IFMAP DB - Invalid state') self._is_ifmap_up = False ConnectionState.update(conn_type = ConnectionType.IFMAP, name = 'IfMap', status = ConnectionStatus.DOWN, message = 'Invalid IFMAP DB State', server_addrs = ["%s:%s" % (self._ifmap_srv_ip, self._ifmap_srv_port)]) finally: gevent.sleep( self._get_api_server().get_ifmap_health_check_interval())
def _sandesh_connection_info_update(self, status, message): new_conn_state = getattr(ConnectionStatus, status) ConnectionState.update(conn_type = ConnectionType.ZOOKEEPER, name = 'Zookeeper', status = new_conn_state, message = message, server_addrs = self._zookeeper_srvr.split(',')) if ((self._conn_state and self._conn_state != ConnectionStatus.DOWN) and new_conn_state == ConnectionStatus.DOWN): msg = 'Connection to Zookeeper down: %s' %(message) self._supress_log(msg) if (self._conn_state and self._conn_state != new_conn_state and new_conn_state == ConnectionStatus.UP): msg = 'Connection to Zookeeper ESTABLISHED' self._supress_log(msg) self._conn_state = new_conn_state
def _get_keystone_conn(self): if self._ks: return if 'v3' in self._auth_url.split('/')[-1]: self._ks = self._ksv3_get_conn() if self._endpoint_type and self._ks.service_catalog: self._ks.management_url = \ self._ks.service_catalog.get_urls( service_type='identity', endpoint_type=self._endpoint_type)[0] else: self._ks = self._ksv2_get_conn() ConnectionState.update(conn_type=ConnType.OTHER, name='Keystone', status=ConnectionStatus.UP, message='', server_addrs=[self._auth_url])
def __init__(self, redis_uve_list, logger, redis_password=None, \ uvedbcache=None, usecache=False, freq=5): self._logger = logger self._redis = None self._uvedbcache = uvedbcache self._usecache = usecache self._redis_password = redis_password self._uve_reverse_map = {} self._freq = freq for h,m in UVE_MAP.iteritems(): self._uve_reverse_map[m] = h # Fill in redis/collector instances self._redis_uve_map = {} for new_elem in redis_uve_list: test_elem = RedisInstKey(ip=new_elem[0], port=new_elem[1]) self._redis_uve_map[test_elem] = RedisInst() ConnectionState.update(ConnectionType.REDIS_UVE,\ test_elem.ip+":"+str(test_elem.port), ConnectionStatus.INIT)
def update_redis_uve_list(self, redis_uve_list): newlist = set(redis_uve_list) # if some redis instances are gone, remove them from our map for test_elem in self._redis_uve_map.keys(): r_ip = test_elem[0] r_port = test_elem[1] redis_inst = (r_ip, int(r_port)) if redis_inst not in newlist: self._redis_uve_map[test_elem].deleted = True else: self._redis_uve_map[test_elem].deleted = False # new redis instances need to be inserted into the map for new_elem in newlist: new_redis = RedisInstKey(ip=new_elem[0], port=new_elem[1]) if new_redis not in self._redis_uve_map: self._redis_uve_map[new_redis] = RedisInst() ConnectionState.update(conn_type = ConnectionType.REDIS_UVE,\ name = new_elem[0]+":"+str(new_elem[1]), status = \ ConnectionStatus.INIT, server_addrs = \ [new_elem[0]+":"+str(new_elem[1])])
def _sandesh_connection_info_update(self, status, message): from pysandesh.connection_info import ConnectionState from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, ConnectionType new_conn_state = getattr(ConnectionStatus, status) ConnectionState.update( conn_type=ConnectionType.ZOOKEEPER, name="Zookeeper", status=new_conn_state, message=message, server_addrs=self._zookeeper_srvr.split(","), ) if (self._conn_state and self._conn_state != ConnectionStatus.DOWN) and new_conn_state == ConnectionStatus.DOWN: msg = "Connection to Zookeeper down: %s" % (message) self._supress_log(msg) if self._conn_state and self._conn_state != new_conn_state and new_conn_state == ConnectionStatus.UP: msg = "Connection to Zookeeper ESTABLISHED" self._supress_log(msg) self._conn_state = new_conn_state
def run(self): ConnectionState.update(conn_type = ConnectionType.REDIS_UVE, name = 'LOCAL', status = ConnectionStatus.INIT) while True: if self._redis: redish = self._redis else: redish = redis.StrictRedis(self._local_redis_uve[0], self._local_redis_uve[1], password=self._redis_password, db=1) try: if not self._redis: value = "" redish.ping() else: k, value = redish.brpop("DELETED") self._logger.debug("%s del received for " % value) # value is of the format: # DEL:<key>:<src>:<node-type>:<module>:<instance-id>:<message-type>:<seqno> redish.delete(value) except: if self._redis: #send redis connection down msg. Coule be bcos of authentication ConnectionState.update(conn_type = ConnectionType.REDIS_UVE, name = 'LOCAL', status = ConnectionStatus.DOWN) self._redis = None gevent.sleep(5) else: self._logger.debug("Deleted %s" % value) if not self._redis: self._redis = redish ConnectionState.update(conn_type = ConnectionType.REDIS_UVE, name = 'LOCAL', status = ConnectionStatus.UP)
def _publish_int(self, service, data, oper_state = 'up', msg = ''): self.syslog('Publish service "%s", data "%s"' % (service, data)) payload = { service : data, 'service-type' : service, 'remote-addr' : self.remote_addr, 'oper-state' : oper_state, 'oper-state-reason' : msg } emsg = None cookie = None try: self.inc_pub_stats(service, 'request') r = requests.post( self.puburl, data=json.dumps(payload), headers=self._headers, timeout=30) if r.status_code != 200: self.inc_pub_stats(service, 'sc_%d' % r.status_code) emsg = 'Status Code ' + str(r.status_code) except requests.exceptions.ConnectionError: self.inc_pub_stats(service, 'conn_error') emsg = 'Connection Error' except requests.exceptions.Timeout: self.inc_pub_stats(service, 'timeout') emsg = 'Request Timeout' finally: ConnectionState.update(conn_type = ConnectionType.DISCOVERY, name = service, status = ConnectionStatus.DOWN if emsg else ConnectionStatus.UP, server_addrs = ['%s:%s' % (self._server_ip, \ self._server_port)], message = 'Publish Error - %s' % emsg if emsg else 'Publish Success') if not emsg: self.inc_pub_stats(service, 'response') self.inc_pub_stats(service, 'blob', value = json.dumps(data)) response = r.json() cookie = response['cookie'] self.pubdata[cookie] = (service, data) self.syslog('Saving token %s' % (cookie)) return cookie