def list_proxies_api():
''' lists twems'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if 'owner' in request.args:
twems = list_twems_by_owner(request['owner'])
elif 'operator' in request.args:
twems = list_twems_by_operator(set(request['operator']))
else:
twems = list_twems()
twem_data = {}
for twem_id in twems:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
twem_data[twem_id] = {
'name': twem.name,
'lb': 'tcp://%s:%s' % (app.config['MONACO_DB'], twem_id),
'servers': twem.servers,
}
except Exception:
twem_data[twem_id] = {'service': monaco.name_by_twem_id[twem_id]}
return jsonify(twem_data)
def __init__(self, twem_id, node_id, host, threshold=3):
threading.Thread.__init__(self)
self.twem_id = twem_id
self.node_id = node_id
self.twem = schema.MonacoTwem(twem_id=twem_id)
self.host = host
self.nutmgr = nutmgmt.NutMgmt()
self.interval = config.config['stats']['interval']
self.threshold = threshold
self.failcount = 0
self.logger = STATLOGGER
self._run = True
def list_twems_by_owner(owner):
''' list all twem_ids where owner == twem.owner '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
twems = []
for twem_id in monaco.twem_ids:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if twem.owner == owner:
twems.append(twem_id)
except Exception:
pass
return twems
def list_twems_by_operator(groups):
''' lists all twems operated by one of the groups '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
twems = []
for twem_id in monaco.twem_ids:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if twem.operator in groups:
twems.append(twem_id)
except Exception:
pass
return twems
def handler(_):
''' handler for when the proxies on this node are modified '''
try:
with self.lock:
self.node.refresh(self.r)
# this will stop, reconf, and start nutcracker.
# should work 80% of the time, all of the time ;)
for twem_id in self.node.twems:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(self.r)
self.nutmanager.update_twem(twem)
for twem_id in self.nutmanager.list_nutcracker_instances():
if not twem_id in self.node.twems:
self.nutmanager.delete_twem(twem_id)
except Exception, exc:
self.logger.exception(exc)
def twem_conf_struct(self, twem, retry=True):
'''
Given a schema.MonacoTwem,
returns the nutcracker config for that proxy in dict form
'''
try:
if type(twem) != schema.MonacoTwem:
twem = schema.MonacoTwem(twem_id=twem)
twem.refresh(self.r)
conf = {}
for key in schema.MonacoTwem.HASH_KEYS:
if hasattr(twem, key) and key in self.CONF_KEYS:
conf[key] = getattr(twem, key)
conf['listen'] = twem.listen
conf['auto_eject_hosts'] = twem.auto_eject_hosts
conf['redis'] = True
conf['servers'] = []
if len(twem.servers) == 1:
# configure to proxy across the master and slaves of a single monaco db, using physical hostnames
app = schema.App(app_id=twem.servers[0])
app.refresh(self.r)
monaco = schema.Monaco()
monaco.refresh(self.r)
for node_id in app.nodes:
node = schema.MonacoNode(node_id=node_id)
node.refresh(self.r)
conf['servers'].append('%s:%s:1' % (node.FQDN, app.port))
else:
# configure to proxy across a set of monaco dbs, using the loadbalanced hostname
for app_id in twem.servers:
conf['servers'].append(
'%s:%s:1' %
(config['loadbalancer']['hostname'], app_id))
# Allow for external servers that are manually specified
if twem.extservers:
for server in twem.extservers:
conf['servers'].append('%s:1' % server)
return {twem.name: conf}
except redis.RedisError, err:
self.r = redis.StrictRedis(port=config['mgmt_port'])
if retry:
return self.twem_conf_struct(twem, retry=False)
else:
self.logger.exception(err)
def proxy_stats(twem_id):
'''
Returns live aggregates for a given proxy
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(twem_id) in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
aggregate_rps = 0
aggregate_connections = 0
if len(twem.servers) == 1:
dbapp = schema.App(app_id=twem.servers[0])
dbapp.refresh(r)
for node_id, _ in dbapp.nodes.iteritems():
appcli = StrictRedis(monaco.hostnames_by_node_id[node_id],
dbapp.port)
info = appcli.info()
if 'instantaneous_ops_per_sec' in info:
aggregate_rps += info['instantaneous_ops_per_sec']
if 'connected_clients' in info:
aggregate_connections += info['connected_clients']
else:
for app_id in twem.servers:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
appcli = dbapp.get_master_connection(r)
info = appcli.info()
if 'instantaneous_ops_per_sec' in info:
aggregate_rps += info['instantaneous_ops_per_sec']
if 'connected_clients' in info:
aggregate_connections += info['connected_clients']
return jsonify({
'total_rps': aggregate_rps,
'total_connections': aggregate_connections
})
def proxy_view(twem_id):
''' Templates the proxy view '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(twem_id) in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
data = {}
data['twem_id'] = twem_id
data['name'] = twem.name
data['servers'] = twem.servers
data['extservers'] = twem.extservers
data['dbinfo'] = {}
for app_id in twem.servers:
# Get usage info on all backend DBs
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
mastercli = dbapp.get_master_connection(r)
info = mastercli.info()
used = float(info['used_memory']) / (1024 * 1024)
total = int(dbapp.maxmemory) // (1024 * 1024)
percent = round((100 * used) / total, 2)
used = round(used, 2)
data['dbinfo'][app_id] = {}
data['dbinfo'][app_id]['total'] = total
data['dbinfo'][app_id]['used'] = used
data['dbinfo'][app_id]['percent'] = percent
data['distribution'] = twem.distribution
data['owner'] = twem.owner
data['operator'] = twem.operator
# choices for servers
data['all_servers'] = [app_id for app_id in list_apps()]
return render_template('proxy.html', **data)
def main():
'''
This is a jazzier version of the node stats reporter.
It will spin up N threads (where N = the number of app Masters on this node)
Those threads will report stats on the config interval
'''
r = redis.StrictRedis(port=config.config['mgmt_port'])
monaco = schema.Monaco()
monaco.refresh(r)
host = config.config['hostname']
node_id = monaco.node_ids_by_hostname[host]
node = schema.MonacoNode(node_id=node_id)
monaco_handler = MonacoHandler(node_id)
monaco_handler.start()
app_threadmap = {}
twem_threadmap = {}
while True:
try:
node.refresh(r)
# Set up this node's master DB handlers
for app_id in app_threadmap.keys():
if app_id not in node.apps:
# child thread should die a natural, painless death
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
for app_id in node.apps:
app = schema.App(app_id=app_id)
app.refresh(r)
if app.nodes[node.node_id] != 'master':
if app_id in app_threadmap:
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
continue
if not app_id in app_threadmap:
# perhaps a new thing
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.debug('started %s', app_id)
elif not app_threadmap[app_id].is_alive():
del app_threadmap[app_id]
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.info('restarted %s', app_id)
# Set up this node's twem handlers
for twem_id in twem_threadmap.keys():
if twem_id not in node.twems:
# child thread should die a natural, painless death
twem_threadmap[twem_id].stop()
del twem_threadmap[twem_id]
STATLOGGER.debug('deleted %s', twem_id)
for twem_id in node.twems:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if not twem_id in twem_threadmap:
# perhaps a new thing
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.debug('started %s', twem_id)
elif not twem_threadmap[twem_id].is_alive():
del twem_threadmap[twem_id]
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.info('restarted %s', twem_id)
except redis.RedisError:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception, exc:
STATLOGGER.exception(exc)
time.sleep(5)
def proxy_api(twem_id):
'''
Twemproxy API:
GET:
200 - gets info about twem in json format
404 - twem_id does not exist
403 - you aint allowed
HEAD:
200 - twem_id exists
404 - twem_id does not exist
POST:
200 - sent update command to master
400 - twem_id does not exist
403 - you aint allowed
DELETE:
200 - sent delete command to master
404 - twem_id does not exist
403 - you aint allowed
'''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
twem_id = str(twem_id)
if request.method == 'HEAD':
if not twem_id in monaco.twem_ids:
abort(404)
return 'OK'
if request.method == 'GET':
if not twem_id in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
data = {}
for key in schema.MonacoTwem.HASH_KEYS:
if hasattr(twem, key):
data[key] = getattr(twem, key)
data['servers'] = twem.servers
data['extservers'] = twem.extservers
return jsonify(data)
if request.method == 'POST':
if twem_id in monaco.twem_ids:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
job = {
'command': 'update_proxy',
'twem_id': twem_id,
}
for key in schema.MonacoTwem.HASH_KEYS:
if key in request.form:
job[key] = request.form[key]
if 'servers' in request.form:
job['servers'] = []
for app_id in request.values.getlist('servers'):
if app_id in monaco.app_ids:
job['servers'].append(app_id)
else:
abort(400)
if 'extservers' in request.form:
job['extservers'] = []
ipport = re.compile(
'^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]{4,5})$'
)
serverport = re.compile('^([a-zA-Z0-9-_]*):([0-9]{4,5})$')
for extserver in request.values.getlist('extservers'):
if ipport.match(extserver):
job['extservers'].append(extserver)
elif serverport.match(extserver):
job['extservers'].append(extserver)
else:
abort(400)
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
else:
# can't create. use POST:/api/proxy
abort(400)
if request.method == 'DELETE':
if not twem_id in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
job = {
'command': 'delete_proxy',
'twem_id': twem_id,
}
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
return 'OK'