def check_clusters_vs_node_apps(testlogger, r):
'''
Check that clusters match node-app sets
'''
result = True
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
app = schema.App(app_id=app_id)
app.refresh(r)
for node_id, _ in app.nodes.iteritems():
if not app_id in r.smembers(schema.NODE_APPS_TMPL % node_id):
testlogger.error(
"App %s has node %s in it's cluster, but the node doesn't have it in it's app-set",
app_id, node_id)
result = False
for node_id in m.node_ids:
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
for app_id in node.apps:
if not node_id in r.hkeys(schema.APP_CLUSTER_TMPL % app_id):
testlogger.error(
"Node %s has app %s in its app-set, but the corresponding app doesn't have the node in it's cluster",
node_id, app_id)
result = False
return result
def __init__(self):
monaco = schema.Monaco()
self.r = redis.StrictRedis(port=config['mgmt_port'])
# terniaries are always a bad idea. this is a mess of exceptions waiting to cascade so FIXME
if self.r.info()['role'] == 'master':
self.rmaster = redis.StrictRedis(port=config['mgmt_port'])
else:
self.rmaster = redis.StrictRedis(host=self.r.info()['master_host'], port=config['mgmt_port'], socket_connect_timeout=1, socket_timeout=1)
monaco.refresh(self.r)
node_id = monaco.node_ids_by_hostname[config['hostname']]
self.node = schema.MonacoNode(node_id=node_id)
self.health_data = {} # dictionary of app_id -> DB health
self.app_clients = {} # dictionary of app_id -> redis clients
self.rps = redis.StrictRedis(port=config['mgmt_port'])
self.pubsub = self.rps.pubsub(ignore_subscribe_messages=True)
self.lock = threading.Lock()
self._subscriptions = {}
self.logger = logging.getLogger('monaco.slave')
self.redmanager = RedisMgmt()
self.nutmanager = NutMgmt()
# for slave based health-checks
self.sched = Scheduler(daemon=True)
self.sched.start()
self.sched.add_interval_job(self.node_health, seconds=5) # TODO: Tune
self.health_check_pool = ThreadPool(10)
atexit.register(lambda: self.sched.shutdown(wait=False))
def app_view(app_id):
''' Web UI for an App '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(app_id) in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
data = {}
for node, role in dbapp.nodes.iteritems():
node = schema.MonacoNode(node_id=node)
node.refresh(r)
if role == 'master':
data[role] = {'host': node.hostname, 'port': dbapp.port}
elif role in data:
data[role].append({'host': node.hostname, 'port': dbapp.port})
else:
data[role] = [{'host': node.hostname, 'port': dbapp.port}]
data['app_id'] = app_id
data['name'] = dbapp.name
# scale bytes to human readable mb/gb
data['maxmemory'] = dbapp.maxmemory
data['maxmemory_policy'] = dbapp.maxmemory_policy
data['persist'] = dbapp.persist == 'True'
data['replicas'] = dbapp.replicas
data['slavelb'] = dbapp.slavelb == 'True'
data['owner'] = dbapp.owner
data['operator'] = dbapp.operator
data['memory_target'] = '&target=monaco.%s.%s.%s.used_memory' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data[
'rps_target'] = '&target=monaco.%s.%s.%s.instantaneous_ops_per_sec' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data['conn_target'] = '&target=monaco.%s.%s.%s.connected_clients' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data['cpu_target'] = '&target=monaco.%s.%s.%s.cpu_percent' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
return render_template('db.html', **data)
def node_api(node_id):
'''
This provides a RESTful endpoint for MonacoNode management
GET- get MonacoNode info
POST- with updated info
DELETE- an existing node to remove from distribution
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if request.method == 'GET':
if not str(node_id) in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=str(node_id))
node.refresh(r)
data = dict(node.__dict__.items() + node.app_info(r).items())
return jsonify(data)
if request.method == 'POST':
if not str(node_id) in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
node.total_memory = request.form['total_memory']
node.rack = request.form['rack']
node.write(r)
return 'OK'
if request.method == 'DELETE':
if str(node_id) in monaco.node_ids:
abort(400)
nodetokill = schema.MonacoNode(node_id=node_id)
if len(nodetokill.apps) != 0 or nodetokill.status != 'MAINTENANCE':
abort(400)
monaco.delete_node(nodetokill, r)
nodetokill.delete(r)
return 'OK'
def twem_conf_struct(self, twem, retry=True):
'''
Given a schema.MonacoTwem,
returns the nutcracker config for that proxy in dict form
'''
try:
if type(twem) != schema.MonacoTwem:
twem = schema.MonacoTwem(twem_id=twem)
twem.refresh(self.r)
conf = {}
for key in schema.MonacoTwem.HASH_KEYS:
if hasattr(twem, key) and key in self.CONF_KEYS:
conf[key] = getattr(twem, key)
conf['listen'] = twem.listen
conf['auto_eject_hosts'] = twem.auto_eject_hosts
conf['redis'] = True
conf['servers'] = []
if len(twem.servers) == 1:
# configure to proxy across the master and slaves of a single monaco db, using physical hostnames
app = schema.App(app_id=twem.servers[0])
app.refresh(self.r)
monaco = schema.Monaco()
monaco.refresh(self.r)
for node_id in app.nodes:
node = schema.MonacoNode(node_id=node_id)
node.refresh(self.r)
conf['servers'].append('%s:%s:1' % (node.FQDN, app.port))
else:
# configure to proxy across a set of monaco dbs, using the loadbalanced hostname
for app_id in twem.servers:
conf['servers'].append(
'%s:%s:1' %
(config['loadbalancer']['hostname'], app_id))
# Allow for external servers that are manually specified
if twem.extservers:
for server in twem.extservers:
conf['servers'].append('%s:1' % server)
return {twem.name: conf}
except redis.RedisError, err:
self.r = redis.StrictRedis(port=config['mgmt_port'])
if retry:
return self.twem_conf_struct(twem, retry=False)
else:
self.logger.exception(err)
def list_nodes_api():
'''
This provides a RESTful endpoint for listing MonacoNodes
HEAD: Returns {'node_ids': [node_id, node_id]}, which only queries master mgmt db
GET: Returns HEAD + {'<node_id>': {NODE INFO DICT}}, which queries redis servers on each node
'''
data = {}
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
data['node_ids'] = monaco.node_ids
for node_id in data['node_ids']:
data[node_id] = {}
data[node_id]['hostname'] = monaco.hostnames_by_node_id[node_id]
if request.method == 'HEAD':
continue
try:
rtemp = StrictRedis(host=data[node_id]['hostname'],
port=app.config['MGMT_PORT'],
socket_connect_timeout=1,
socket_timeout=0.5)
info = rtemp.info()
data[node_id]['role'] = info['role']
if data[node_id]['role'] == 'slave':
data[node_id]['role_details'] = {'master': info['master_host']}
else:
data[node_id]['role_details'] = {}
data[node_id]['role_details']['connected_slaves'] = info[
'connected_slaves']
for idx in xrange(int(info['connected_slaves'])):
data[node_id]['role_details']['slave%d' %
idx] = info['slave%d' % idx]
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
data[node_id]['mem_usage'] = '%sM' % node.app_info(r)['memory']
data[node_id]['up'] = True
except Exception:
data[node_id]['up'] = False
data[node_id]['role'] = None
data[node_id]['role_details'] = None
data[node_id]['mem_usage'] = None
data[node_id]['net_usage'] = None
return jsonify(data)
def node_view(node_id):
''' Web view for MonacoNode '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
node_id = str(node_id)
if not node_id in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
appinfo = node.app_info(r)
data = {
'node_id':
node_id,
'hostname':
node.hostname,
'FQDN':
node.FQDN,
'total_memory':
node.total_memory,
'rack':
node.rack,
'status':
node.status,
'used_memory':
appinfo['memory'],
'memory_percent':
round(100.0 * appinfo['memory'] / (int(node.total_memory) / 2.0), 2),
'master_servers':
len(appinfo['masters']),
'masters':
map(str, sorted(map(int, appinfo['masters']))),
'slave_servers':
len(appinfo['slaves']),
'slaves':
map(str, sorted(map(int, appinfo['slaves']))),
'twemproxy_servers':
len(node.twems),
# General info
'nodes':
map(str, sorted(map(int, monaco.node_ids))),
}
return render_template("node.html", **data)
def new_node_api():
'''
Create new nodes by POST'ing info to this endpoint.
Get a listing of nodes
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if request.method == 'POST':
if str(request.form['node_id']) in monaco.node_ids:
abort(400)
newnode = schema.MonacoNode(node_id=request.form['node_id'])
newnode.apps = []
newnode.twems = []
newnode.hostname = request.form['hostname']
newnode.FQDN = request.form['FQDN']
newnode.total_memory = request.form['total_memory']
newnode.rack = request.form['rack']
newnode.write(r)
monaco.new_node(newnode, r)
return 'OK'
def list_app_api():
''' lists apps'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if 'owner' in request.args:
apps = list_apps_by_owner(request['owner'])
elif 'operator' in request.args:
apps = list_apps_by_operator(set(request['operator']))
else:
apps = list_apps()
app_data = {}
for app_id in apps:
try:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
masternode = schema.MonacoNode(node_id=dbapp.master)
masternode.refresh(r)
mastercli = dbapp.get_master_connection(r)
info = mastercli.info()
used = float(info['used_memory']) / (1024 * 1024)
total = int(dbapp.maxmemory) // (1024 * 1024)
percent = round((100 * used) / total, 2)
used = round(used, 2)
app_data[app_id] = {
'service': dbapp.name,
'exposure': 'tcp://%s:%s' % (masternode.FQDN, dbapp.port),
'memory_used': used,
'memory_total': total,
'memory_percent': percent,
'connected_clients': info['connected_clients'],
'rps': info['instantaneous_ops_per_sec'],
}
except Exception:
app_data[app_id] = {'service': monaco.service_by_app_id[app_id]}
return jsonify(app_data)
def main():
'''
This is a jazzier version of the node stats reporter.
It will spin up N threads (where N = the number of app Masters on this node)
Those threads will report stats on the config interval
'''
r = redis.StrictRedis(port=config.config['mgmt_port'])
monaco = schema.Monaco()
monaco.refresh(r)
host = config.config['hostname']
node_id = monaco.node_ids_by_hostname[host]
node = schema.MonacoNode(node_id=node_id)
monaco_handler = MonacoHandler(node_id)
monaco_handler.start()
app_threadmap = {}
twem_threadmap = {}
while True:
try:
node.refresh(r)
# Set up this node's master DB handlers
for app_id in app_threadmap.keys():
if app_id not in node.apps:
# child thread should die a natural, painless death
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
for app_id in node.apps:
app = schema.App(app_id=app_id)
app.refresh(r)
if app.nodes[node.node_id] != 'master':
if app_id in app_threadmap:
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
continue
if not app_id in app_threadmap:
# perhaps a new thing
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.debug('started %s', app_id)
elif not app_threadmap[app_id].is_alive():
del app_threadmap[app_id]
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.info('restarted %s', app_id)
# Set up this node's twem handlers
for twem_id in twem_threadmap.keys():
if twem_id not in node.twems:
# child thread should die a natural, painless death
twem_threadmap[twem_id].stop()
del twem_threadmap[twem_id]
STATLOGGER.debug('deleted %s', twem_id)
for twem_id in node.twems:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if not twem_id in twem_threadmap:
# perhaps a new thing
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.debug('started %s', twem_id)
elif not twem_threadmap[twem_id].is_alive():
del twem_threadmap[twem_id]
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.info('restarted %s', twem_id)
except redis.RedisError:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception, exc:
STATLOGGER.exception(exc)
time.sleep(5)
def setUp(self):
self.monaconode = schema.MonacoNode(node_id=1)
def app_api(app_id):
'''
App API:
GET:
200 - gets info about app in json format
404 - app_id does not exist
403 - you aint allowed
HEAD:
200 - app_id exists
404 - app_id does not exist
POST:
200 - sent update command to master
400 - app_id does not exist
403 - you aint allowed
DELETE:
200 - sent delete command to master
404 - app_id does not exist
403 - you aint allowed
'''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
app_id = str(app_id)
if request.method == 'HEAD':
if not app_id in monaco.app_ids:
abort(404)
return 'OK'
if request.method == 'GET':
if not app_id in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
app_info = {
'app_id': dbapp.app_id,
'port': dbapp.port,
'nodes': [],
'unused_nodes': [],
}
for k in dbapp.HASH_KEYS:
if hasattr(dbapp, k):
app_info[k] = getattr(dbapp, k)
for node_id, role in dbapp.nodes.iteritems():
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
app_info['nodes'].append({
'host': node.hostname,
'node_id': node_id,
'role': role
})
app_info['unused_nodes'] = [
node_id for node_id in monaco.node_ids
if not node_id in dbapp.nodes
]
return jsonify(app_info)
if request.method == 'POST':
if app_id in monaco.app_ids:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
if request.form['name'] != monaco.service_by_app_id[app_id]:
monaco.rename_app(app_id, request.form['name'], r)
job = {
'command': 'update',
'app_id': app_id,
}
for k in schema.App.HASH_KEYS:
if k in request.form:
job[k] = request.form[k]
if 'persist' in job:
job['persist'] = True
else:
job['persist'] = False
if 'slavelb' in job:
job['slavelb'] = True
else:
job['slavelb'] = False
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
else:
# can't create with an app_id pre-specified.
abort(400)
if request.method == 'DELETE':
if not app_id in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
job = {
'command': 'delete',
'app_id': app_id,
}
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)