class DiscoveryServer():
def __init__(self, args):
self._homepage_links = []
self._args = args
self.service_config = args.service_config
self._debug = {
'hb_stray': 0,
'msg_pubs': 0,
'msg_subs': 0,
'msg_query': 0,
'msg_hbt': 0,
'ttl_short': 0,
'policy_rr': 0,
'policy_lb': 0,
'policy_fi': 0,
'db_upd_hb': 0,
'throttle_subs':0,
'503': 0,
}
self._ts_use = 1
self.short_ttl_map = {}
self._sem = BoundedSemaphore(1)
self._base_url = "http://%s:%s" % (self._args.listen_ip_addr,
self._args.listen_port)
self._pipe_start_app = None
bottle.route('/', 'GET', self.homepage_http_get)
# heartbeat
bottle.route('/heartbeat', 'POST', self.api_heartbeat)
# publish service
bottle.route('/publish', 'POST', self.api_publish)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/publish', 'publish service'))
bottle.route('/publish/<end_point>', 'POST', self.api_publish)
# subscribe service
bottle.route('/subscribe', 'POST', self.api_subscribe)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/subscribe', 'subscribe service'))
# query service
bottle.route('/query', 'POST', self.api_query)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/query', 'query service'))
# collection - services
bottle.route('/services', 'GET', self.show_all_services)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/services', 'show published services'))
bottle.route('/services.json', 'GET', self.services_json)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/services.json',
'List published services in JSON format'))
# show a specific service type
bottle.route('/services/<service_type>', 'GET', self.show_all_services)
# update service
bottle.route('/service/<id>', 'PUT', self.service_http_put)
# get service info
bottle.route('/service/<id>', 'GET', self.service_http_get)
bottle.route('/service/<id>/brief', 'GET', self.service_brief_http_get)
# delete (un-publish) service
bottle.route('/service/<id>', 'DELETE', self.service_http_delete)
# collection - clients
bottle.route('/clients', 'GET', self.show_all_clients)
bottle.route('/clients/<service_type>/<service_id>', 'GET', self.show_all_clients)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/clients', 'list all subscribers'))
bottle.route('/clients.json', 'GET', self.clients_json)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/clients.json',
'list all subscribers in JSON format'))
# show config
bottle.route('/config', 'GET', self.show_config)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/config', 'show discovery service config'))
# show debug
bottle.route('/stats', 'GET', self.show_stats)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/stats', 'show discovery service stats'))
# cleanup
bottle.route('/cleanup', 'GET', self.cleanup_http_get)
self._homepage_links.append(LinkObject('action',
self._base_url , '/cleanup', 'Purge inactive publishers'))
if not self._pipe_start_app:
self._pipe_start_app = bottle.app()
# sandesh init
self._sandesh = Sandesh()
module = Module.DISCOVERY_SERVICE
module_name = ModuleNames[module]
node_type = Module2NodeType[module]
node_type_name = NodeTypeNames[node_type]
instance_id = self._args.worker_id
self._sandesh.init_generator(
module_name, socket.gethostname(), node_type_name, instance_id,
self._args.collectors, 'discovery_context',
int(self._args.http_server_port), ['sandesh'])
self._sandesh.set_logging_params(enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level,
file=self._args.log_file)
self._sandesh.trace_buffer_create(name="dsHeartBeatTraceBuf",
size=1000)
# DB interface initialization
self._db_connect(self._args.reset_config)
# build in-memory subscriber data
self._sub_data = {}
for (client_id, service_type) in self._db_conn.subscriber_entries():
self.create_sub_data(client_id, service_type)
# end __init__
def create_sub_data(self, client_id, service_type):
if not client_id in self._sub_data:
self._sub_data[client_id] = {}
if not service_type in self._sub_data[client_id]:
sdata = {
'ttl_expires': 0,
'heartbeat': int(time.time()),
}
self._sub_data[client_id][service_type] = sdata
return self._sub_data[client_id][service_type]
# end
def delete_sub_data(self, client_id, service_type):
if (client_id in self._sub_data and
service_type in self._sub_data[client_id]):
del self._sub_data[client_id][service_type]
if len(self._sub_data[client_id]) == 0:
del self._sub_data[client_id]
# end
def get_sub_data(self, id, service_type):
if id in self._sub_data and service_type in self._sub_data[id]:
return self._sub_data[id][service_type]
return self.create_sub_data(id, service_type)
# end
# Public Methods
def get_args(self):
return self._args
# end get_args
def get_ip_addr(self):
return self._args.listen_ip_addr
# end get_ip_addr
def get_port(self):
return self._args.listen_port
# end get_port
def get_pipe_start_app(self):
return self._pipe_start_app
# end get_pipe_start_app
def homepage_http_get(self):
json_links = []
url = bottle.request.url[:-1]
for link in self._homepage_links:
json_links.append({'link': link.to_dict(with_url=url)})
json_body = \
{"href": self._base_url,
"links": json_links
}
return json_body
# end homepage_http_get
def get_service_config(self, service_type, item):
service = service_type.lower()
if (service in self.service_config and
item in self.service_config[service]):
return self.service_config[service][item]
elif item in self._args.__dict__:
return self._args.__dict__[item]
else:
return None
# end
def _db_connect(self, reset_config):
self._db_conn = DiscoveryCassendraClient("discovery",
self._args.cassandra_server_list, reset_config)
# end _db_connect
def cleanup(self):
pass
# end cleanup
def syslog(self, log_msg):
log = sandesh.discServiceLog(
log_msg=log_msg, sandesh=self._sandesh)
log.send(sandesh=self._sandesh)
def get_ttl_short(self, client_id, service_type, default):
ttl = default
if not client_id in self.short_ttl_map:
self.short_ttl_map[client_id] = {}
if service_type in self.short_ttl_map[client_id]:
# keep doubling till we land in normal range
ttl = self.short_ttl_map[client_id][service_type] * 2
if ttl >= 32:
ttl = 32
self.short_ttl_map[client_id][service_type] = ttl
return ttl
# end
# check if service expired (return color along)
def service_expired(self, entry, include_color=False, include_down=True):
timedelta = datetime.timedelta(
seconds=(int(time.time()) - entry['heartbeat']))
if self._args.hc_interval <= 0:
# health check has been disabled
color = "#00FF00" # green - all good
expired = False
elif timedelta.seconds <= self._args.hc_interval:
color = "#00FF00" # green - all good
expired = False
elif (timedelta.seconds > (self._args.hc_interval *
self._args.hc_max_miss)):
color = "#FF0000" # red - publication expired
expired = True
else:
color = "#FFFF00" # yellow - missed some heartbeats
expired = False
if include_down and entry['admin_state'] != 'up':
color = "#FF0000" # red - publication expired
expired = True
if include_color:
return (expired, color, timedelta)
else:
return expired
# end service_expired
# decorator to catch DB error
def db_error_handler(func):
def error_handler(*args, **kwargs):
try:
return func(*args,**kwargs)
except disc_exceptions.ServiceUnavailable:
self._debug['503'] += 1
bottle.abort(503, 'Service Unavailable')
except Exception as e:
raise
return error_handler
# 404 forces republish
def heartbeat(self, sig):
# self.syslog('heartbeat from "%s"' % sig)
self._debug['msg_hbt'] += 1
info = sig.split(':')
if len(info) != 2:
self.syslog('Unable to parse heartbeat cookie %s' % sig)
bottle.abort(404, 'Unable to parse heartbeat')
service_type = info[1]
service_id = info[0]
entry = self._db_conn.lookup_service(service_type, service_id)
if not entry:
self.syslog('Received stray heartbeat with cookie %s' % (sig))
self._debug['hb_stray'] += 1
bottle.abort(404, 'Publisher %s not found' % sig)
# update heartbeat timestamp in database
entry['heartbeat'] = int(time.time())
# insert entry if timed out by background task
self._db_conn.update_service(
service_type, entry['service_id'], entry)
m = sandesh.dsHeartBeat(
publisher_id=sig, service_type=service_type,
sandesh=self._sandesh)
m.trace_msg(name='dsHeartBeatTraceBuf', sandesh=self._sandesh)
return '200 OK'
# end heartbeat
@db_error_handler
def api_heartbeat(self):
ctype = bottle.request.headers['content-type']
json_req = {}
try:
if ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
else:
data = bottle.request.json
except Exception as e:
self.syslog('Unable to parse heartbeat')
self.syslog(bottle.request.body.buf)
bottle.abort(400, 'Unable to parse heartbeat')
status = self.heartbeat(data['cookie'])
return status
@db_error_handler
def api_publish(self, end_point = None):
self._debug['msg_pubs'] += 1
ctype = bottle.request.headers['content-type']
json_req = {}
if ctype == 'application/json':
data = bottle.request.json
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = info
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = dict(info)
else:
bottle.abort(400, e)
sig = end_point or publisher_id(
bottle.request.environ['REMOTE_ADDR'], json.dumps(json_req))
# Rx {'name': u'ifmap-server', 'info': {u'ip_addr': u'10.84.7.1',
# u'port': u'8443'}}
info = json_req['info']
service_type = json_req['name']
entry = self._db_conn.lookup_service(service_type, service_id=sig)
if not entry:
entry = {
'service_type': service_type,
'service_id': sig,
'in_use': 0,
'ts_use': 0,
'ts_created': int(time.time()),
'prov_state': 'new',
'remote': bottle.request.environ.get('REMOTE_ADDR'),
'sequence': str(int(time.time())) + socket.gethostname(),
}
self._db_conn.insert_service(service_type, sig, entry)
entry['info'] = info
entry['admin_state'] = 'up'
entry['heartbeat'] = int(time.time())
# insert entry if new or timed out
self._db_conn.update_service(service_type, sig, entry)
response = {'cookie': sig + ':' + service_type}
if ctype != 'application/json':
response = xmltodict.unparse({'response': response})
self.syslog('publish service "%s", sid=%s, info=%s'
% (service_type, sig, info))
if not service_type.lower() in self.service_config:
self.service_config[
service_type.lower()] = self._args.default_service_opts
return response
# end api_publish
# round-robin
def service_list_round_robin(self, pubs):
self._debug['policy_rr'] += 1
return sorted(pubs, key=lambda service: service['ts_use'])
# end
# load-balance
def service_list_load_balance(self, pubs):
self._debug['policy_lb'] += 1
return sorted(pubs, key=lambda service: service['in_use'])
# end
# master election
def service_list_fixed(self, pubs):
self._debug['policy_fi'] += 1
return sorted(pubs, key=lambda service: service['sequence'])
# end
def service_list(self, service_type, pubs):
policy = self.get_service_config(service_type, 'policy') or 'load-balance'
if policy == 'load-balance':
f = self.service_list_load_balance
elif policy == 'fixed':
f = self.service_list_fixed
else:
f = self.service_list_round_robin
return f(pubs)
# end
@db_error_handler
def api_subscribe(self):
self._debug['msg_subs'] += 1
ctype = bottle.request.headers['content-type']
if ctype == 'application/json':
json_req = bottle.request.json
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
json_req = {}
for service_type, info in data.items():
json_req['service'] = service_type
json_req.update(dict(info))
else:
bottle.abort(400, e)
service_type = json_req['service']
client_id = json_req['client']
count = reqcnt = int(json_req['instances'])
client_type = json_req.get('client-type', '')
assigned_sid = set()
r = []
ttl = randint(self._args.ttl_min, self._args.ttl_max)
# check client entry and any existing subscriptions
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
if not cl_entry:
cl_entry = {
'instances': count,
'remote': bottle.request.environ.get('REMOTE_ADDR'),
'client_type': client_type,
}
self.create_sub_data(client_id, service_type)
self._db_conn.insert_client_data(service_type, client_id, cl_entry)
sdata = self.get_sub_data(client_id, service_type)
if sdata:
sdata['ttl_expires'] += 1
# send short ttl if no publishers
pubs = self._db_conn.lookup_service(service_type) or []
pubs_active = [item for item in pubs if not self.service_expired(item)]
if len(pubs_active) == 0:
ttl_short = self.get_service_config(service_type, 'ttl_short')
if ttl_short:
ttl = self.get_ttl_short( client_id, service_type, ttl_short)
self._debug['ttl_short'] += 1
self.syslog(
'subscribe: service type=%s, client=%s:%s, ttl=%d, asked=%d pubs=%d/%d, subs=%d'
% (service_type, client_type, client_id, ttl, count,
len(pubs), len(pubs_active), len(subs)))
if subs:
plist = dict((entry['service_id'],entry) for entry in pubs_active)
for service_id, result in subs:
# previously published service is gone
entry = plist.get(service_id, None)
if entry is None:
continue
result = entry['info']
self._db_conn.insert_client(
service_type, service_id, client_id, result, ttl)
r.append(result)
assigned_sid.add(service_id)
count -= 1
if count == 0:
response = {'ttl': ttl, service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# find least loaded instances
pubs = self.service_list(service_type, pubs_active)
# prepare response - send all if count 0
for entry in pubs:
# skip duplicates - could happen if some publishers have quit and
# we have already picked up others from cached information above
if entry['service_id'] in assigned_sid:
continue
assigned_sid.add(entry['service_id'])
result = entry['info']
r.append(result)
self.syslog(' assign service=%s, info=%s' %
(entry['service_id'], json.dumps(result)))
# create client entry
self._db_conn.insert_client(
service_type, entry['service_id'], client_id, result, ttl)
# update publisher entry
entry['ts_use'] = self._ts_use
self._ts_use += 1
self._db_conn.update_service(
service_type, entry['service_id'], entry)
count -= 1
if count == 0:
break
response = {'ttl': ttl, service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# end api_subscribe
def api_query(self):
self._debug['msg_query'] += 1
ctype = bottle.request.headers['content-type']
if ctype == 'application/json':
json_req = bottle.request.json
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
json_req = {}
for service_type, info in data.items():
json_req['service'] = service_type
json_req.update(dict(info))
else:
bottle.abort(400, e)
service_type = json_req['service']
count = int(json_req['instances'])
r = []
# lookup publishers of the service
pubs = self._db_conn.query_service(service_type)
if not pubs:
return {service_type: r}
# eliminate inactive services
pubs_active = [item for item in pubs if not self.service_expired(item)]
self.syslog(' query: Found %s publishers, %d active, need %d' %
(len(pubs), len(pubs_active), count))
# find least loaded instances
pubs = pubs_active
# prepare response - send all if count 0
for index in range(min(count, len(pubs)) if count else len(pubs)):
entry = pubs[index]
result = entry['info']
r.append(result)
self.syslog(' assign service=%s, info=%s' %
(entry['service_id'], json.dumps(result)))
response = {service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# end api_subscribe
@db_error_handler
def show_all_services(self, service_type=None):
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
rsp += ' <tr>\n'
rsp += ' <td>Service Type</td>\n'
rsp += ' <td>Remote IP</td>\n'
rsp += ' <td>Service Id</td>\n'
rsp += ' <td>Provision State</td>\n'
rsp += ' <td>Admin State</td>\n'
rsp += ' <td>In Use</td>\n'
rsp += ' <td>Time since last Heartbeat</td>\n'
rsp += ' </tr>\n'
for pub in self._db_conn.service_entries(service_type):
info = pub['info']
service_type = pub['service_type']
service_id = pub['service_id']
sig = service_id + ':' + service_type
rsp += ' <tr>\n'
if service_type:
rsp += ' <td>' + service_type + '</td>\n'
else:
link = do_html_url("/services/%s" % service_type, service_type)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + pub['remote'] + '</td>\n'
link = do_html_url("/service/%s/brief" % sig, sig)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + pub['prov_state'] + '</td>\n'
rsp += ' <td>' + pub['admin_state'] + '</td>\n'
link = do_html_url("/clients/%s/%s" % (service_type, service_id),
str(pub['in_use']))
rsp += ' <td>' + link + '</td>\n'
(expired, color, timedelta) = self.service_expired(
pub, include_color=True)
#status = "down" if expired else "up"
rsp += ' <td bgcolor=%s>' % (
color) + str(timedelta) + '</td>\n'
rsp += ' </tr>\n'
rsp += ' </table>\n'
return rsp
# end show_services
@db_error_handler
def services_json(self, service_type=None):
rsp = []
for pub in self._db_conn.service_entries(service_type):
entry = pub.copy()
entry['status'] = "down" if self.service_expired(entry) else "up"
entry['hbcount'] = 0
# send unique service ID (hash or service endpoint + type)
entry['service_id'] = str(entry['service_id'] + ':' + entry['service_type'])
rsp.append(entry)
return {'services': rsp}
# end services_json
def service_http_put(self, id):
self.syslog('Update service %s' % (id))
try:
json_req = bottle.request.json
service_type = json_req['service_type']
self.syslog('Entry %s' % (json_req))
except (ValueError, KeyError, TypeError) as e:
bottle.abort(400, e)
entry = self._db_conn.lookup_service(service_type, service_id=id)
if not entry:
bottle.abort(405, 'Unknown service')
if 'admin_state' in json_req:
entry['admin_state'] = json_req['admin_state']
self._db_conn.update_service(service_type, id, entry)
self.syslog('update service=%s, sid=%s, info=%s'
% (service_type, id, entry))
return {}
# end service_http_put
def service_http_delete(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
self.syslog('Delete service %s:%s' % (service_id, service_type))
entry = self._db_conn.lookup_service(service_type, service_id)
if not entry:
bottle.abort(405, 'Unknown service')
entry['admin_state'] = 'down'
self._db_conn.update_service(service_type, service_id, entry)
self.syslog('delete service=%s, sid=%s, info=%s'
% (service_type, service_id, entry))
return {}
# end service_http_delete
# return service info - meta as well as published data
def service_http_get(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
pub = self._db_conn.lookup_service(service_type, service_id)
if pub:
entry = pub.copy()
entry['hbcount'] = 0
entry['status'] = "down" if self.service_expired(entry) else "up"
return entry
# end service_http_get
# return service info - only published data
def service_brief_http_get(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
entry = self._db_conn.lookup_service(service_type, service_id)
if entry:
return entry['info']
else:
return 'Unknown service %s' % id
# end service_brief_http_get
# purge expired publishers
def cleanup_http_get(self):
for entry in self._db_conn.service_entries():
if self.service_expired(entry):
self._db_conn.delete_service(entry)
return self.show_all_services()
#end
@db_error_handler
def show_all_clients(self, service_type=None, service_id=None):
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
rsp += ' <tr>\n'
rsp += ' <td>Client IP</td>\n'
rsp += ' <td>Client Type</td>\n'
rsp += ' <td>Client Id</td>\n'
rsp += ' <td>Service Type</td>\n'
rsp += ' <td>Service Id</td>\n'
rsp += ' <td>TTL (sec)</td>\n'
rsp += ' <td>Time Remaining</td>\n'
rsp += ' <td>Refresh Count</td>\n'
rsp += ' </tr>\n'
# lookup subscribers of the service
clients = self._db_conn.get_all_clients(service_type=service_type,
service_id=service_id)
if not clients:
return rsp
for client in clients:
(service_type, client_id, service_id, mtime, ttl) = client
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
if cl_entry is None:
continue
sdata = self.get_sub_data(client_id, service_type)
if sdata is None:
self.syslog('Missing sdata for client %s, service %s' %
(client_id, service_type))
continue
rsp += ' <tr>\n'
rsp += ' <td>' + cl_entry['remote'] + '</td>\n'
client_type = cl_entry.get('client_type', '')
rsp += ' <td>' + client_type + '</td>\n'
rsp += ' <td>' + client_id + '</td>\n'
rsp += ' <td>' + service_type + '</td>\n'
sig = service_id + ':' + service_type
link = do_html_url("service/%s/brief" % (sig), sig)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + str(ttl) + '</td>\n'
remaining = ttl - int(time.time() - mtime)
rsp += ' <td>' + str(remaining) + '</td>\n'
rsp += ' <td>' + str(sdata['ttl_expires']) + '</td>\n'
rsp += ' </tr>\n'
rsp += ' </table>\n'
return rsp
# end show_clients
@db_error_handler
def clients_json(self):
rsp = []
clients = self._db_conn.get_all_clients()
if not clients:
return {'services': rsp}
for client in clients:
(service_type, client_id, service_id, mtime, ttl) = client
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
sdata = self.get_sub_data(client_id, service_type)
if sdata is None:
self.syslog('Missing sdata for client %s, service %s' %
(client_id, service_type))
continue
entry = cl_entry.copy()
entry.update(sdata)
entry['client_id'] = str(client_id)
entry['service_type'] = str(service_type)
# send unique service ID (hash or service endpoint + type)
entry['service_id'] = str(service_id + ':' + service_type)
entry['ttl'] = ttl
rsp.append(entry)
return {'services': rsp}
# end show_clients
def show_config(self):
"""
r = {}
r['global'] = self._args.__dict__
for service, config in self.service_config.items():
r[service] = config
return r
"""
rsp = output.display_user_menu()
#rsp += '<h4>Defaults:</h4>'
rsp += '<table border="1" cellpadding="1" cellspacing="0">\n'
rsp += '<tr><th colspan="2">Defaults</th></tr>'
for k in sorted(self._args.__dict__.iterkeys()):
v = self._args.__dict__[k]
rsp += '<tr><td>%s</td><td>%s</td></tr>' % (k, v)
rsp += '</table>'
rsp += '<br>'
for service, config in self.service_config.items():
#rsp += '<h4>%s:</h4>' %(service)
rsp += '<table border="1" cellpadding="1" cellspacing="0">\n'
rsp += '<tr><th colspan="2">%s</th></tr>' % (service)
for k in sorted(config.iterkeys()):
rsp += '<tr><td>%s</td><td>%s</td></tr>' % (k, config[k])
rsp += '</table>'
rsp += '<br>'
return rsp
# end show_config
def show_stats(self):
stats = self._debug
stats.update(self._db_conn.get_debug_stats())
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
for k in sorted(stats.iterkeys()):
rsp += ' <tr>\n'
rsp += ' <td>%s</td>\n' % (k)
rsp += ' <td>%s</td>\n' % (stats[k])
rsp += ' </tr>\n'
return rsp
class DiscoveryServer():
def __init__(self, args):
self._homepage_links = []
self._args = args
self.service_config = args.service_config
self._debug = {
'hb_stray': 0,
'msg_pubs': 0,
'msg_subs': 0,
'msg_query': 0,
'msg_hbt': 0,
'ttl_short': 0,
'policy_rr': 0,
'policy_lb': 0,
'policy_fi': 0,
'db_upd_hb': 0,
'throttle_subs':0,
}
self._ts_use = 1
self.short_ttl_map = {}
self._sem = BoundedSemaphore(1)
self._base_url = "http://%s:%s" % (self._args.listen_ip_addr,
self._args.listen_port)
self._pipe_start_app = None
bottle.route('/', 'GET', self.homepage_http_get)
# heartbeat
bottle.route('/heartbeat', 'POST', self.api_heartbeat)
# publish service
bottle.route('/publish', 'POST', self.api_publish)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/publish', 'publish service'))
bottle.route('/publish/<end_point>', 'POST', self.api_publish)
# subscribe service
bottle.route('/subscribe', 'POST', self.api_subscribe)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/subscribe', 'subscribe service'))
# query service
bottle.route('/query', 'POST', self.api_query)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/query', 'query service'))
# collection - services
bottle.route('/services', 'GET', self.show_all_services)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/services', 'show published services'))
bottle.route('/services.json', 'GET', self.services_json)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/services.json',
'List published services in JSON format'))
# show a specific service type
bottle.route('/services/<service_type>', 'GET', self.show_all_services)
# update service
bottle.route('/service/<id>', 'PUT', self.service_http_put)
# get service info
bottle.route('/service/<id>', 'GET', self.service_http_get)
bottle.route('/service/<id>/brief', 'GET', self.service_brief_http_get)
# delete (un-publish) service
bottle.route('/service/<id>', 'DELETE', self.service_http_delete)
# collection - clients
bottle.route('/clients', 'GET', self.show_all_clients)
bottle.route('/clients/<service_type>/<service_id>', 'GET', self.show_all_clients)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/clients', 'list all subscribers'))
bottle.route('/clients.json', 'GET', self.clients_json)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/clients.json',
'list all subscribers in JSON format'))
# show config
bottle.route('/config', 'GET', self.show_config)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/config', 'show discovery service config'))
# show debug
bottle.route('/stats', 'GET', self.show_stats)
self._homepage_links.append(
LinkObject(
'action',
self._base_url , '/stats', 'show discovery service stats'))
# cleanup
bottle.route('/cleanup', 'GET', self.cleanup_http_get)
self._homepage_links.append(LinkObject('action',
self._base_url , '/cleanup', 'Purge inactive publishers'))
if not self._pipe_start_app:
self._pipe_start_app = bottle.app()
# sandesh init
self._sandesh = Sandesh()
module = Module.DISCOVERY_SERVICE
module_name = ModuleNames[module]
node_type = Module2NodeType[module]
node_type_name = NodeTypeNames[node_type]
instance_id = self._args.worker_id
self._sandesh.init_generator(
module_name, socket.gethostname(), node_type_name, instance_id,
self._args.collectors, 'discovery_context',
int(self._args.http_server_port), ['sandesh'])
self._sandesh.set_logging_params(enable_local_log=self._args.log_local,
category=self._args.log_category,
level=self._args.log_level,
file=self._args.log_file)
self._sandesh.trace_buffer_create(name="dsHeartBeatTraceBuf",
size=1000)
# DB interface initialization
self._db_connect(self._args.reset_config)
# build in-memory subscriber data
self._sub_data = {}
for (client_id, service_type) in self._db_conn.subscriber_entries():
self.create_sub_data(client_id, service_type)
# end __init__
def create_sub_data(self, client_id, service_type):
if not client_id in self._sub_data:
self._sub_data[client_id] = {}
if not service_type in self._sub_data[client_id]:
sdata = {
'ttl_expires': 0,
'heartbeat': int(time.time()),
}
self._sub_data[client_id][service_type] = sdata
return self._sub_data[client_id][service_type]
# end
def delete_sub_data(self, client_id, service_type):
if (client_id in self._sub_data and
service_type in self._sub_data[client_id]):
del self._sub_data[client_id][service_type]
if len(self._sub_data[client_id]) == 0:
del self._sub_data[client_id]
# end
def get_sub_data(self, id, service_type):
if id in self._sub_data and service_type in self._sub_data[id]:
return self._sub_data[id][service_type]
return self.create_sub_data(id, service_type)
# end
# Public Methods
def get_args(self):
return self._args
# end get_args
def get_ip_addr(self):
return self._args.listen_ip_addr
# end get_ip_addr
def get_port(self):
return self._args.listen_port
# end get_port
def get_pipe_start_app(self):
return self._pipe_start_app
# end get_pipe_start_app
def homepage_http_get(self):
json_links = []
url = bottle.request.url[:-1]
for link in self._homepage_links:
json_links.append({'link': link.to_dict(with_url=url)})
json_body = \
{"href": self._base_url,
"links": json_links
}
return json_body
# end homepage_http_get
def get_service_config(self, service_type, item):
service = service_type.lower()
if (service in self.service_config and
item in self.service_config[service]):
return self.service_config[service][item]
elif item in self._args.__dict__:
return self._args.__dict__[item]
else:
return None
# end
def _db_connect(self, reset_config):
self._db_conn = DiscoveryCassendraClient("discovery",
self._args.cassandra_server_list, reset_config)
# end _db_connect
def cleanup(self):
pass
# end cleanup
def syslog(self, log_msg):
log = sandesh.discServiceLog(
log_msg=log_msg, sandesh=self._sandesh)
log.send(sandesh=self._sandesh)
def get_ttl_short(self, client_id, service_type, default):
ttl = default
if not client_id in self.short_ttl_map:
self.short_ttl_map[client_id] = {}
if service_type in self.short_ttl_map[client_id]:
# keep doubling till we land in normal range
ttl = self.short_ttl_map[client_id][service_type] * 2
if ttl >= 32:
ttl = 32
self.short_ttl_map[client_id][service_type] = ttl
return ttl
# end
# check if service expired (return color along)
def service_expired(self, entry, include_color=False, include_down=True):
timedelta = datetime.timedelta(
seconds=(int(time.time()) - entry['heartbeat']))
if self._args.hc_interval <= 0:
# health check has been disabled
color = "#00FF00" # green - all good
expired = False
elif timedelta.seconds <= self._args.hc_interval:
color = "#00FF00" # green - all good
expired = False
elif (timedelta.seconds > (self._args.hc_interval *
self._args.hc_max_miss)):
color = "#FF0000" # red - publication expired
expired = True
else:
color = "#FFFF00" # yellow - missed some heartbeats
expired = False
if include_down and entry['admin_state'] != 'up':
color = "#FF0000" # red - publication expired
expired = True
if include_color:
return (expired, color, timedelta)
else:
return expired
# end service_expired
# decorator to catch DB error
def db_error_handler(func):
def error_handler(*args, **kwargs):
try:
return func(*args,**kwargs)
except disc_exceptions.ServiceUnavailable:
bottle.abort(503, 'Service Unavailable')
except Exception as e:
raise
return error_handler
# 404 forces republish
def heartbeat(self, sig):
# self.syslog('heartbeat from "%s"' % sig)
self._debug['msg_hbt'] += 1
info = sig.split(':')
if len(info) != 2:
self.syslog('Unable to parse heartbeat cookie %s' % sig)
bottle.abort(404, 'Unable to parse heartbeat')
service_type = info[1]
service_id = info[0]
entry = self._db_conn.lookup_service(service_type, service_id)
if not entry:
self.syslog('Received stray heartbeat with cookie %s' % (sig))
self._debug['hb_stray'] += 1
bottle.abort(404, 'Publisher %s not found' % sig)
# update heartbeat timestamp in database
entry['heartbeat'] = int(time.time())
# insert entry if timed out by background task
self._db_conn.update_service(
service_type, entry['service_id'], entry)
m = sandesh.dsHeartBeat(
publisher_id=sig, service_type=service_type,
sandesh=self._sandesh)
m.trace_msg(name='dsHeartBeatTraceBuf', sandesh=self._sandesh)
return '200 OK'
# end heartbeat
@db_error_handler
def api_heartbeat(self):
ctype = bottle.request.headers['content-type']
json_req = {}
try:
if ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
else:
data = bottle.request.json
except Exception as e:
self.syslog('Unable to parse heartbeat')
self.syslog(bottle.request.body.buf)
bottle.abort(400, 'Unable to parse heartbeat')
status = self.heartbeat(data['cookie'])
return status
@db_error_handler
def api_publish(self, end_point = None):
self._debug['msg_pubs'] += 1
ctype = bottle.request.headers['content-type']
json_req = {}
if ctype == 'application/json':
data = bottle.request.json
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = info
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = dict(info)
else:
bottle.abort(400, e)
sig = end_point or publisher_id(
bottle.request.environ['REMOTE_ADDR'], json.dumps(json_req))
# Rx {'name': u'ifmap-server', 'info': {u'ip_addr': u'10.84.7.1',
# u'port': u'8443'}}
info = json_req['info']
service_type = json_req['name']
entry = self._db_conn.lookup_service(service_type, service_id=sig)
if not entry:
entry = {
'service_type': service_type,
'service_id': sig,
'in_use': 0,
'ts_use': 0,
'ts_created': int(time.time()),
'prov_state': 'new',
'remote': bottle.request.environ.get('REMOTE_ADDR'),
'sequence': service_type+sig
}
self._db_conn.insert_service(service_type, sig, entry)
entry['info'] = info
entry['admin_state'] = 'up'
entry['heartbeat'] = int(time.time())
# insert entry if new or timed out
self._db_conn.update_service(service_type, sig, entry)
response = {'cookie': sig + ':' + service_type}
if ctype != 'application/json':
response = xmltodict.unparse({'response': response})
self.syslog('publish service "%s", sid=%s, info=%s'
% (service_type, sig, info))
if not service_type.lower() in self.service_config:
self.service_config[
service_type.lower()] = self._args.default_service_opts
return response
# end api_publish
# round-robin
def service_list_round_robin(self, pubs):
self._debug['policy_rr'] += 1
return sorted(pubs, key=lambda service: service['ts_use'])
# end
# load-balance
def service_list_load_balance(self, pubs):
self._debug['policy_lb'] += 1
return sorted(pubs, key=lambda service: service['in_use'])
# end
# master election
def service_list_fixed(self, pubs):
self._debug['policy_fi'] += 1
return sorted(pubs, key=lambda service: service['sequence'])
# end
def service_list(self, service_type, pubs):
policy = self.get_service_config(service_type, 'policy') or 'load-balance'
if policy == 'load-balance':
f = self.service_list_load_balance
elif policy == 'fixed':
f = self.service_list_fixed
else:
f = self.service_list_round_robin
return f(pubs)
# end
@db_error_handler
def api_subscribe(self):
self._debug['msg_subs'] += 1
ctype = bottle.request.headers['content-type']
if ctype == 'application/json':
json_req = bottle.request.json
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
json_req = {}
for service_type, info in data.items():
json_req['service'] = service_type
json_req.update(dict(info))
else:
bottle.abort(400, e)
service_type = json_req['service']
client_id = json_req['client']
count = reqcnt = int(json_req['instances'])
client_type = json_req.get('client-type', '')
assigned_sid = set()
r = []
ttl = randint(self._args.ttl_min, self._args.ttl_max)
# check client entry and any existing subscriptions
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
if not cl_entry:
cl_entry = {
'instances': count,
'remote': bottle.request.environ.get('REMOTE_ADDR'),
'client_type': client_type,
}
self.create_sub_data(client_id, service_type)
self._db_conn.insert_client_data(service_type, client_id, cl_entry)
sdata = self.get_sub_data(client_id, service_type)
if sdata:
sdata['ttl_expires'] += 1
# send short ttl if no publishers
pubs = self._db_conn.lookup_service(service_type) or []
pubs_active = [item for item in pubs if not self.service_expired(item)]
if len(pubs_active) == 0:
ttl_short = self.get_service_config(service_type, 'ttl_short')
if ttl_short:
ttl = self.get_ttl_short( client_id, service_type, ttl_short)
self._debug['ttl_short'] += 1
self.syslog(
'subscribe: service type=%s, client=%s:%s, ttl=%d, asked=%d pubs=%d/%d, subs=%d'
% (service_type, client_type, client_id, ttl, count,
len(pubs), len(pubs_active), len(subs)))
if subs:
plist = dict((entry['service_id'],entry) for entry in pubs_active)
for service_id, result in subs:
# previously published service is gone
entry = plist.get(service_id, None)
if entry is None:
continue
result = entry['info']
self._db_conn.insert_client(
service_type, service_id, client_id, result, ttl)
r.append(result)
assigned_sid.add(service_id)
count -= 1
if count == 0:
response = {'ttl': ttl, service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# find least loaded instances
pubs = self.service_list(service_type, pubs_active)
# prepare response - send all if count 0
for entry in pubs:
# skip duplicates - could happen if some publishers have quit and
# we have already picked up others from cached information above
if entry['service_id'] in assigned_sid:
continue
assigned_sid.add(entry['service_id'])
result = entry['info']
r.append(result)
self.syslog(' assign service=%s, info=%s' %
(entry['service_id'], json.dumps(result)))
# create client entry
self._db_conn.insert_client(
service_type, entry['service_id'], client_id, result, ttl)
# update publisher entry
entry['ts_use'] = self._ts_use
self._ts_use += 1
self._db_conn.update_service(
service_type, entry['service_id'], entry)
count -= 1
if count == 0:
break
response = {'ttl': ttl, service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# end api_subscribe
def api_query(self):
self._debug['msg_query'] += 1
ctype = bottle.request.headers['content-type']
if ctype == 'application/json':
json_req = bottle.request.json
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
json_req = {}
for service_type, info in data.items():
json_req['service'] = service_type
json_req.update(dict(info))
else:
bottle.abort(400, e)
service_type = json_req['service']
count = int(json_req['instances'])
r = []
# lookup publishers of the service
pubs = self._db_conn.query_service(service_type)
if not pubs:
return {service_type: r}
# eliminate inactive services
pubs_active = [item for item in pubs if not self.service_expired(item)]
self.syslog(' query: Found %s publishers, %d active, need %d' %
(len(pubs), len(pubs_active), count))
# find least loaded instances
pubs = pubs_active
# prepare response - send all if count 0
for index in range(min(count, len(pubs)) if count else len(pubs)):
entry = pubs[index]
result = entry['info']
r.append(result)
self.syslog(' assign service=%s, info=%s' %
(entry['service_id'], json.dumps(result)))
response = {service_type: r}
if ctype == 'application/xml':
response = xmltodict.unparse({'response': response})
return response
# end api_subscribe
@db_error_handler
def show_all_services(self, service_type=None):
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
rsp += ' <tr>\n'
rsp += ' <td>Service Type</td>\n'
rsp += ' <td>Remote IP</td>\n'
rsp += ' <td>Service Id</td>\n'
rsp += ' <td>Provision State</td>\n'
rsp += ' <td>Admin State</td>\n'
rsp += ' <td>In Use</td>\n'
rsp += ' <td>Time since last Heartbeat</td>\n'
rsp += ' </tr>\n'
for pub in self._db_conn.service_entries(service_type):
info = pub['info']
service_type = pub['service_type']
service_id = pub['service_id']
sig = service_id + ':' + service_type
rsp += ' <tr>\n'
if service_type:
rsp += ' <td>' + service_type + '</td>\n'
else:
link = do_html_url("/services/%s" % service_type, service_type)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + pub['remote'] + '</td>\n'
link = do_html_url("/service/%s/brief" % sig, sig)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + pub['prov_state'] + '</td>\n'
rsp += ' <td>' + pub['admin_state'] + '</td>\n'
link = do_html_url("/clients/%s/%s" % (service_type, service_id),
str(pub['in_use']))
rsp += ' <td>' + link + '</td>\n'
(expired, color, timedelta) = self.service_expired(
pub, include_color=True)
#status = "down" if expired else "up"
rsp += ' <td bgcolor=%s>' % (
color) + str(timedelta) + '</td>\n'
rsp += ' </tr>\n'
rsp += ' </table>\n'
return rsp
# end show_services
@db_error_handler
def services_json(self, service_type=None):
rsp = []
for pub in self._db_conn.service_entries(service_type):
entry = pub.copy()
entry['status'] = "down" if self.service_expired(entry) else "up"
entry['hbcount'] = 0
# send unique service ID (hash or service endpoint + type)
entry['service_id'] = str(entry['service_id'] + ':' + entry['service_type'])
rsp.append(entry)
return {'services': rsp}
# end services_json
def service_http_put(self, id):
self.syslog('Update service %s' % (id))
try:
json_req = bottle.request.json
service_type = json_req['service_type']
self.syslog('Entry %s' % (json_req))
except (ValueError, KeyError, TypeError) as e:
bottle.abort(400, e)
entry = self._db_conn.lookup_service(service_type, service_id=id)
if not entry:
bottle.abort(405, 'Unknown service')
if 'admin_state' in json_req:
entry['admin_state'] = json_req['admin_state']
self._db_conn.update_service(service_type, id, entry)
self.syslog('update service=%s, sid=%s, info=%s'
% (service_type, id, entry))
return {}
# end service_http_put
def service_http_delete(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
self.syslog('Delete service %s:%s' % (service_id, service_type))
entry = self._db_conn.lookup_service(service_type, service_id)
if not entry:
bottle.abort(405, 'Unknown service')
entry['admin_state'] = 'down'
self._db_conn.update_service(service_type, service_id, entry)
self.syslog('delete service=%s, sid=%s, info=%s'
% (service_type, service_id, entry))
return {}
# end service_http_delete
# return service info - meta as well as published data
def service_http_get(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
pub = self._db_conn.lookup_service(service_type, service_id)
if pub:
entry = pub.copy()
entry['hbcount'] = 0
entry['status'] = "down" if self.service_expired(entry) else "up"
return entry
# end service_http_get
# return service info - only published data
def service_brief_http_get(self, id):
info = id.split(':')
service_type = info[1]
service_id = info[0]
entry = self._db_conn.lookup_service(service_type, service_id)
if entry:
return entry['info']
else:
return 'Unknown service %s' % id
# end service_brief_http_get
# purge expired publishers
def cleanup_http_get(self):
for entry in self._db_conn.service_entries():
if self.service_expired(entry):
self._db_conn.delete_service(entry)
return self.show_all_services()
#end
@db_error_handler
def show_all_clients(self, service_type=None, service_id=None):
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
rsp += ' <tr>\n'
rsp += ' <td>Client IP</td>\n'
rsp += ' <td>Client Type</td>\n'
rsp += ' <td>Client Id</td>\n'
rsp += ' <td>Service Type</td>\n'
rsp += ' <td>Service Id</td>\n'
rsp += ' <td>TTL (sec)</td>\n'
rsp += ' <td>Time Remaining</td>\n'
rsp += ' <td>Refresh Count</td>\n'
rsp += ' </tr>\n'
# lookup subscribers of the service
clients = self._db_conn.get_all_clients(service_type=service_type,
service_id=service_id)
if not clients:
return rsp
for client in clients:
(service_type, client_id, service_id, mtime, ttl) = client
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
if cl_entry is None:
continue
sdata = self.get_sub_data(client_id, service_type)
if sdata is None:
self.syslog('Missing sdata for client %s, service %s' %
(client_id, service_type))
continue
rsp += ' <tr>\n'
rsp += ' <td>' + cl_entry['remote'] + '</td>\n'
client_type = cl_entry.get('client_type', '')
rsp += ' <td>' + client_type + '</td>\n'
rsp += ' <td>' + client_id + '</td>\n'
rsp += ' <td>' + service_type + '</td>\n'
sig = service_id + ':' + service_type
link = do_html_url("service/%s/brief" % (sig), sig)
rsp += ' <td>' + link + '</td>\n'
rsp += ' <td>' + str(ttl) + '</td>\n'
remaining = ttl - int(time.time() - mtime)
rsp += ' <td>' + str(remaining) + '</td>\n'
rsp += ' <td>' + str(sdata['ttl_expires']) + '</td>\n'
rsp += ' </tr>\n'
rsp += ' </table>\n'
return rsp
# end show_clients
@db_error_handler
def clients_json(self):
rsp = []
clients = self._db_conn.get_all_clients()
if not clients:
return {'services': rsp}
for client in clients:
(service_type, client_id, service_id, mtime, ttl) = client
cl_entry, subs = self._db_conn.lookup_client(service_type, client_id)
sdata = self.get_sub_data(client_id, service_type)
if sdata is None:
self.syslog('Missing sdata for client %s, service %s' %
(client_id, service_type))
continue
entry = cl_entry.copy()
entry.update(sdata)
entry['client_id'] = str(client_id)
entry['service_type'] = str(service_type)
# send unique service ID (hash or service endpoint + type)
entry['service_id'] = str(service_id + ':' + service_type)
entry['ttl'] = ttl
rsp.append(entry)
return {'services': rsp}
# end show_clients
def show_config(self):
"""
r = {}
r['global'] = self._args.__dict__
for service, config in self.service_config.items():
r[service] = config
return r
"""
rsp = output.display_user_menu()
#rsp += '<h4>Defaults:</h4>'
rsp += '<table border="1" cellpadding="1" cellspacing="0">\n'
rsp += '<tr><th colspan="2">Defaults</th></tr>'
for k in sorted(self._args.__dict__.iterkeys()):
v = self._args.__dict__[k]
rsp += '<tr><td>%s</td><td>%s</td></tr>' % (k, v)
rsp += '</table>'
rsp += '<br>'
for service, config in self.service_config.items():
#rsp += '<h4>%s:</h4>' %(service)
rsp += '<table border="1" cellpadding="1" cellspacing="0">\n'
rsp += '<tr><th colspan="2">%s</th></tr>' % (service)
for k in sorted(config.iterkeys()):
rsp += '<tr><td>%s</td><td>%s</td></tr>' % (k, config[k])
rsp += '</table>'
rsp += '<br>'
return rsp
# end show_config
def show_stats(self):
stats = self._debug
stats.update(self._db_conn.get_debug_stats())
rsp = output.display_user_menu()
rsp += ' <table border="1" cellpadding="1" cellspacing="0">\n'
for k in sorted(stats.iterkeys()):
rsp += ' <tr>\n'
rsp += ' <td>%s</td>\n' % (k)
rsp += ' <td>%s</td>\n' % (stats[k])
rsp += ' </tr>\n'
return rsp
def _db_connect(self, reset_config):
self._db_conn = DiscoveryCassendraClient("discovery",
self._args.cassandra_server_list, reset_config)
def _db_connect(self, reset_config):
self._db_conn = DiscoveryCassendraClient("discovery",
self._args.cassandra_server_list, reset_config)
