def new_proxy_api():
''' for creating new proxies '''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
if request.form['name'] in monaco.twem_ids_by_name:
abort(400)
job = {
'command': 'new_proxy',
}
for k in schema.MonacoTwem.HASH_KEYS:
if k in request.form:
job[k] = request.form[k]
job['servers'] = []
for app_id in request.values.getlist('servers'):
if app_id in monaco.app_ids:
job['servers'].append(app_id)
else:
abort(400)
job['extservers'] = []
ipport = re.compile(
'^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]{4,5})$')
serverport = re.compile('^([a-zA-Z0-9_-]*):([0-9]{4,5})$')
for extserver in request.values.getlist('extservers'):
if ipport.match(extserver):
job['extservers'].append(extserver)
elif serverport.match(extserver):
job['extservers'].append(extserver)
else:
abort(400)
job['hash'] = 'murmur'
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
def __init__(self):
monaco = schema.Monaco()
self.r = redis.StrictRedis(port=config['mgmt_port'])
# terniaries are always a bad idea. this is a mess of exceptions waiting to cascade so FIXME
if self.r.info()['role'] == 'master':
self.rmaster = redis.StrictRedis(port=config['mgmt_port'])
else:
self.rmaster = redis.StrictRedis(host=self.r.info()['master_host'], port=config['mgmt_port'], socket_connect_timeout=1, socket_timeout=1)
monaco.refresh(self.r)
node_id = monaco.node_ids_by_hostname[config['hostname']]
self.node = schema.MonacoNode(node_id=node_id)
self.health_data = {} # dictionary of app_id -> DB health
self.app_clients = {} # dictionary of app_id -> redis clients
self.rps = redis.StrictRedis(port=config['mgmt_port'])
self.pubsub = self.rps.pubsub(ignore_subscribe_messages=True)
self.lock = threading.Lock()
self._subscriptions = {}
self.logger = logging.getLogger('monaco.slave')
self.redmanager = RedisMgmt()
self.nutmanager = NutMgmt()
# for slave based health-checks
self.sched = Scheduler(daemon=True)
self.sched.start()
self.sched.add_interval_job(self.node_health, seconds=5) # TODO: Tune
self.health_check_pool = ThreadPool(10)
atexit.register(lambda: self.sched.shutdown(wait=False))
def list_proxies_api():
''' lists twems'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if 'owner' in request.args:
twems = list_twems_by_owner(request['owner'])
elif 'operator' in request.args:
twems = list_twems_by_operator(set(request['operator']))
else:
twems = list_twems()
twem_data = {}
for twem_id in twems:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
twem_data[twem_id] = {
'name': twem.name,
'lb': 'tcp://%s:%s' % (app.config['MONACO_DB'], twem_id),
'servers': twem.servers,
}
except Exception:
twem_data[twem_id] = {'service': monaco.name_by_twem_id[twem_id]}
return jsonify(twem_data)
def check_clusters_vs_node_apps(testlogger, r):
'''
Check that clusters match node-app sets
'''
result = True
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
app = schema.App(app_id=app_id)
app.refresh(r)
for node_id, _ in app.nodes.iteritems():
if not app_id in r.smembers(schema.NODE_APPS_TMPL % node_id):
testlogger.error(
"App %s has node %s in it's cluster, but the node doesn't have it in it's app-set",
app_id, node_id)
result = False
for node_id in m.node_ids:
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
for app_id in node.apps:
if not node_id in r.hkeys(schema.APP_CLUSTER_TMPL % app_id):
testlogger.error(
"Node %s has app %s in its app-set, but the corresponding app doesn't have the node in it's cluster",
node_id, app_id)
result = False
return result
def new_app_api():
''' for creating new apps '''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
if request.form['name'] in monaco.app_ids_by_service:
abort(400)
job = {
'command': 'new',
}
for k in schema.App.HASH_KEYS:
if k in request.form:
job[k] = request.form[k]
if not 'persist' in request.form:
job['persist'] = False
else:
job['persist'] = True
if not 'slavelb' in request.form:
job['slavelb'] = False
else:
job['slavelb'] = True
jid = job_queue.pushback_job(job)
if not jid:
# just retry in app
abort(503)
return jsonify(jid=jid)
def check_apps_and_clusters(testlogger, r):
'''
Check that apps:clusters is 1:1
'''
result = True
app_hashes = [
key for key in r.keys(schema.APP_HASH_TMPL % '*')
if not (key.endswith('cluster') or key.endswith('status')
or key.endswith('version'))
]
app_clusters = r.keys(schema.APP_CLUSTER_TMPL % '*')
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
hashkey = schema.APP_HASH_TMPL % app_id
clusterkey = schema.APP_CLUSTER_TMPL % app_id
if hashkey in app_hashes:
app_hashes.remove(hashkey)
else:
testlogger.error("App %s doesn't have a corresponding app hash",
app_id)
result = False
if clusterkey in app_clusters:
app_clusters.remove(clusterkey)
else:
testlogger.error(
"App %s doesn't have a corresponding cluster hash", app_id)
result = False
if len(app_hashes) != 0:
testlogger.warn('Extra app hashes: %s', app_hashes)
result = False
if len(app_clusters) != 0:
testlogger.warn('Extra cluster hashes: %s', app_clusters)
result = False
return result
def app_redis_api(app_id):
'''
Simple REST api to redis DBs.
GET, PUT, DELETE are key operations, and POST allows for any command
method = 'GET': ../redis?key=key
return r.get('key')
method = 'PUT': ../redis?key=key&val=val
return r.set('key', 'val')
method = 'DELETE': ../redis?key=key
return r.delete('key')
method = 'POST': ../redis?cmd=hset&args=key,hashkey,hashval
Request args
return getattr(r,cmd)(*args.split(','))
aka r.hset(key, hashkey, hashval)
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
app_id = str(app_id)
if not app_id in monaco.app_ids:
abort(404)
userapp = schema.App(app_id=app_id)
userapp.refresh(r)
master_host = None
for node_id, role in userapp.nodes.iteritems():
if role == 'master':
master_host = monaco.hostnames_by_node_id[node_id]
break
assert master_host != None
r = StrictRedis(master_host, userapp.port)
if request.method == 'GET':
if 'key' not in request.args:
abort(400)
return r.get(request.args['key'])
if request.method == 'PUT':
if 'key' not in request.args or 'val' not in request.args:
abort(400)
return r.set(request.args['key'], request.args['val'])
if request.method == 'DELETE':
if 'key' not in request.args:
abort(400)
return r.delete(request.args['key'])
if request.method == 'POST':
if 'cmd' not in request.args or not hasattr(r, request.args['cmd']):
abort(400)
if 'args' in request.args:
args = request.args['args'].split(',')
else:
args = []
return getattr(r, request.args['cmd'])(*args)
abort(400)
def app_view(app_id):
''' Web UI for an App '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(app_id) in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
data = {}
for node, role in dbapp.nodes.iteritems():
node = schema.MonacoNode(node_id=node)
node.refresh(r)
if role == 'master':
data[role] = {'host': node.hostname, 'port': dbapp.port}
elif role in data:
data[role].append({'host': node.hostname, 'port': dbapp.port})
else:
data[role] = [{'host': node.hostname, 'port': dbapp.port}]
data['app_id'] = app_id
data['name'] = dbapp.name
# scale bytes to human readable mb/gb
data['maxmemory'] = dbapp.maxmemory
data['maxmemory_policy'] = dbapp.maxmemory_policy
data['persist'] = dbapp.persist == 'True'
data['replicas'] = dbapp.replicas
data['slavelb'] = dbapp.slavelb == 'True'
data['owner'] = dbapp.owner
data['operator'] = dbapp.operator
data['memory_target'] = '&target=monaco.%s.%s.%s.used_memory' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data[
'rps_target'] = '&target=monaco.%s.%s.%s.instantaneous_ops_per_sec' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data['conn_target'] = '&target=monaco.%s.%s.%s.connected_clients' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
data['cpu_target'] = '&target=monaco.%s.%s.%s.cpu_percent' % (
app.config['ENV'],
app.config['LOCATION'],
app_id,
)
return render_template('db.html', **data)
def list_apps_by_owner(owner):
''' list all apps where owner == app.owner '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
apps = []
for app_id in monaco.app_ids:
try:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
if dbapp.owner == owner:
apps.append(app_id)
except Exception:
pass
return apps
def list_twems_by_owner(owner):
''' list all twem_ids where owner == twem.owner '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
twems = []
for twem_id in monaco.twem_ids:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if twem.owner == owner:
twems.append(twem_id)
except Exception:
pass
return twems
def list_apps_by_operator(groups):
''' lists all apps operated by one of the groups '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
apps = []
for app_id in monaco.app_ids:
try:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
if dbapp.operator in groups:
apps.append(app_id)
except Exception:
pass
return apps
def list_twems_by_operator(groups):
''' lists all twems operated by one of the groups '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
twems = []
for twem_id in monaco.twem_ids:
try:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if twem.operator in groups:
twems.append(twem_id)
except Exception:
pass
return twems
def validate_app_invariences(testlogger, r):
'''
Ensure that the user specified replica count is maintained
'''
result = True
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
app = schema.App(app_id=app_id)
app.refresh(r)
if int(app.replicas) != len(app.nodes):
testlogger.error("App %s doesn't have the desired replica count",
app_id)
result = False
return result
def check_allocated_nodes_online(testlogger, r):
'''
Ensure that any node allocated to hold a redis instance for an app is online
NOTE: This only verifies against the reported 'status'.
'''
result = True
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
for node_id, _ in r.hgetall(schema.APP_CLUSTER_TMPL %
app_id).iteritems():
if r.hget(schema.NODE_HASH_TMPL % node_id, 'status') != 'UP':
testlogger.error(
"App %s has node %s allocated to host an instance, despite the node being marked DOWN."
% (app_id, node_id))
result = False
return result
def twem_conf_struct(self, twem, retry=True):
'''
Given a schema.MonacoTwem,
returns the nutcracker config for that proxy in dict form
'''
try:
if type(twem) != schema.MonacoTwem:
twem = schema.MonacoTwem(twem_id=twem)
twem.refresh(self.r)
conf = {}
for key in schema.MonacoTwem.HASH_KEYS:
if hasattr(twem, key) and key in self.CONF_KEYS:
conf[key] = getattr(twem, key)
conf['listen'] = twem.listen
conf['auto_eject_hosts'] = twem.auto_eject_hosts
conf['redis'] = True
conf['servers'] = []
if len(twem.servers) == 1:
# configure to proxy across the master and slaves of a single monaco db, using physical hostnames
app = schema.App(app_id=twem.servers[0])
app.refresh(self.r)
monaco = schema.Monaco()
monaco.refresh(self.r)
for node_id in app.nodes:
node = schema.MonacoNode(node_id=node_id)
node.refresh(self.r)
conf['servers'].append('%s:%s:1' % (node.FQDN, app.port))
else:
# configure to proxy across a set of monaco dbs, using the loadbalanced hostname
for app_id in twem.servers:
conf['servers'].append(
'%s:%s:1' %
(config['loadbalancer']['hostname'], app_id))
# Allow for external servers that are manually specified
if twem.extservers:
for server in twem.extservers:
conf['servers'].append('%s:1' % server)
return {twem.name: conf}
except redis.RedisError, err:
self.r = redis.StrictRedis(port=config['mgmt_port'])
if retry:
return self.twem_conf_struct(twem, retry=False)
else:
self.logger.exception(err)
def list_nodes_api():
'''
This provides a RESTful endpoint for listing MonacoNodes
HEAD: Returns {'node_ids': [node_id, node_id]}, which only queries master mgmt db
GET: Returns HEAD + {'<node_id>': {NODE INFO DICT}}, which queries redis servers on each node
'''
data = {}
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
data['node_ids'] = monaco.node_ids
for node_id in data['node_ids']:
data[node_id] = {}
data[node_id]['hostname'] = monaco.hostnames_by_node_id[node_id]
if request.method == 'HEAD':
continue
try:
rtemp = StrictRedis(host=data[node_id]['hostname'],
port=app.config['MGMT_PORT'],
socket_connect_timeout=1,
socket_timeout=0.5)
info = rtemp.info()
data[node_id]['role'] = info['role']
if data[node_id]['role'] == 'slave':
data[node_id]['role_details'] = {'master': info['master_host']}
else:
data[node_id]['role_details'] = {}
data[node_id]['role_details']['connected_slaves'] = info[
'connected_slaves']
for idx in xrange(int(info['connected_slaves'])):
data[node_id]['role_details']['slave%d' %
idx] = info['slave%d' % idx]
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
data[node_id]['mem_usage'] = '%sM' % node.app_info(r)['memory']
data[node_id]['up'] = True
except Exception:
data[node_id]['up'] = False
data[node_id]['role'] = None
data[node_id]['role_details'] = None
data[node_id]['mem_usage'] = None
data[node_id]['net_usage'] = None
return jsonify(data)
def node_view(node_id):
''' Web view for MonacoNode '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
node_id = str(node_id)
if not node_id in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
appinfo = node.app_info(r)
data = {
'node_id':
node_id,
'hostname':
node.hostname,
'FQDN':
node.FQDN,
'total_memory':
node.total_memory,
'rack':
node.rack,
'status':
node.status,
'used_memory':
appinfo['memory'],
'memory_percent':
round(100.0 * appinfo['memory'] / (int(node.total_memory) / 2.0), 2),
'master_servers':
len(appinfo['masters']),
'masters':
map(str, sorted(map(int, appinfo['masters']))),
'slave_servers':
len(appinfo['slaves']),
'slaves':
map(str, sorted(map(int, appinfo['slaves']))),
'twemproxy_servers':
len(node.twems),
# General info
'nodes':
map(str, sorted(map(int, monaco.node_ids))),
}
return render_template("node.html", **data)
def proxy_stats(twem_id):
'''
Returns live aggregates for a given proxy
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(twem_id) in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
aggregate_rps = 0
aggregate_connections = 0
if len(twem.servers) == 1:
dbapp = schema.App(app_id=twem.servers[0])
dbapp.refresh(r)
for node_id, _ in dbapp.nodes.iteritems():
appcli = StrictRedis(monaco.hostnames_by_node_id[node_id],
dbapp.port)
info = appcli.info()
if 'instantaneous_ops_per_sec' in info:
aggregate_rps += info['instantaneous_ops_per_sec']
if 'connected_clients' in info:
aggregate_connections += info['connected_clients']
else:
for app_id in twem.servers:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
appcli = dbapp.get_master_connection(r)
info = appcli.info()
if 'instantaneous_ops_per_sec' in info:
aggregate_rps += info['instantaneous_ops_per_sec']
if 'connected_clients' in info:
aggregate_connections += info['connected_clients']
return jsonify({
'total_rps': aggregate_rps,
'total_connections': aggregate_connections
})
def new_node_api():
'''
Create new nodes by POST'ing info to this endpoint.
Get a listing of nodes
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if request.method == 'POST':
if str(request.form['node_id']) in monaco.node_ids:
abort(400)
newnode = schema.MonacoNode(node_id=request.form['node_id'])
newnode.apps = []
newnode.twems = []
newnode.hostname = request.form['hostname']
newnode.FQDN = request.form['FQDN']
newnode.total_memory = request.form['total_memory']
newnode.rack = request.form['rack']
newnode.write(r)
monaco.new_node(newnode, r)
return 'OK'
def list_app_api():
''' lists apps'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if 'owner' in request.args:
apps = list_apps_by_owner(request['owner'])
elif 'operator' in request.args:
apps = list_apps_by_operator(set(request['operator']))
else:
apps = list_apps()
app_data = {}
for app_id in apps:
try:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
masternode = schema.MonacoNode(node_id=dbapp.master)
masternode.refresh(r)
mastercli = dbapp.get_master_connection(r)
info = mastercli.info()
used = float(info['used_memory']) / (1024 * 1024)
total = int(dbapp.maxmemory) // (1024 * 1024)
percent = round((100 * used) / total, 2)
used = round(used, 2)
app_data[app_id] = {
'service': dbapp.name,
'exposure': 'tcp://%s:%s' % (masternode.FQDN, dbapp.port),
'memory_used': used,
'memory_total': total,
'memory_percent': percent,
'connected_clients': info['connected_clients'],
'rps': info['instantaneous_ops_per_sec'],
}
except Exception:
app_data[app_id] = {'service': monaco.service_by_app_id[app_id]}
return jsonify(app_data)
def node_api(node_id):
'''
This provides a RESTful endpoint for MonacoNode management
GET- get MonacoNode info
POST- with updated info
DELETE- an existing node to remove from distribution
'''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if request.method == 'GET':
if not str(node_id) in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=str(node_id))
node.refresh(r)
data = dict(node.__dict__.items() + node.app_info(r).items())
return jsonify(data)
if request.method == 'POST':
if not str(node_id) in monaco.node_ids:
abort(404)
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
node.total_memory = request.form['total_memory']
node.rack = request.form['rack']
node.write(r)
return 'OK'
if request.method == 'DELETE':
if str(node_id) in monaco.node_ids:
abort(400)
nodetokill = schema.MonacoNode(node_id=node_id)
if len(nodetokill.apps) != 0 or nodetokill.status != 'MAINTENANCE':
abort(400)
monaco.delete_node(nodetokill, r)
nodetokill.delete(r)
return 'OK'
def proxy_view(twem_id):
''' Templates the proxy view '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
if not str(twem_id) in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
data = {}
data['twem_id'] = twem_id
data['name'] = twem.name
data['servers'] = twem.servers
data['extservers'] = twem.extservers
data['dbinfo'] = {}
for app_id in twem.servers:
# Get usage info on all backend DBs
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
mastercli = dbapp.get_master_connection(r)
info = mastercli.info()
used = float(info['used_memory']) / (1024 * 1024)
total = int(dbapp.maxmemory) // (1024 * 1024)
percent = round((100 * used) / total, 2)
used = round(used, 2)
data['dbinfo'][app_id] = {}
data['dbinfo'][app_id]['total'] = total
data['dbinfo'][app_id]['used'] = used
data['dbinfo'][app_id]['percent'] = percent
data['distribution'] = twem.distribution
data['owner'] = twem.owner
data['operator'] = twem.operator
# choices for servers
data['all_servers'] = [app_id for app_id in list_apps()]
return render_template('proxy.html', **data)
def check_all_nodes_in_nodes_list(testlogger, r):
'''
Same as above, but for nodes
'''
m = schema.Monaco()
m.refresh(r)
getid = re.compile(r'-([0-9]*)$')
noextra = True
for node in r.keys(schema.NODE_HASH_TMPL % '*'):
match = getid.search(node)
if not match:
continue
node = match.group(1)
if node in m.node_ids:
m.node_ids.remove(node)
else:
testlogger.warn('Extra node struct found at key: %s' % node)
noextra = False
for node in m.node_ids:
testlogger.error(
'Node %s found in top-level node list, but no corresponding struct found'
% node)
return len(m.node_ids) == 0 and noextra
def check_cluster_sanity(testlogger, r):
'''
Check that each cluster has only 1 master
'''
result = True
m = schema.Monaco()
m.refresh(r)
for app_id in m.app_ids:
cluster = r.hgetall(schema.APP_CLUSTER_TMPL % app_id)
master = [k for k, v in cluster.iteritems() if v == 'master']
if len(master) != 1:
testlogger.error('App %s has %s masters!' % (app_id, len(master)))
result = False
invalid = [
k for k, v in cluster.iteritems()
if not v in ['master', 'slave', 'slave-write']
]
if len(invalid) > 0:
testlogger.error('App %s has an invalid cluster specification: %s',
app_id, cluster)
result = False
return result
def check_all_apps_in_apps_list(testlogger, r):
'''
Check if the monaco-apps set represents the set of monaco app hashes in r
'''
m = schema.Monaco()
m.refresh(r)
getid = re.compile(r'-([0-9]*)$')
noextra = True
# mix the template with a wildcard to match all
for app in r.keys(schema.APP_HASH_TMPL % '*'):
match = getid.search(app)
if not match:
continue
app = match.group(1)
if app in m.app_ids:
m.app_ids.remove(app)
else:
testlogger.warn('Extra app found at key: %s' % app)
noextra = False
for app in m.app_ids:
testlogger.error(
'App %s found in top-level app list, but no corresponding struct found'
% app)
return len(m.app_ids) == 0 and noextra
def list_apps():
''' returns all app_ids '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
return monaco.app_ids
def app_api(app_id):
'''
App API:
GET:
200 - gets info about app in json format
404 - app_id does not exist
403 - you aint allowed
HEAD:
200 - app_id exists
404 - app_id does not exist
POST:
200 - sent update command to master
400 - app_id does not exist
403 - you aint allowed
DELETE:
200 - sent delete command to master
404 - app_id does not exist
403 - you aint allowed
'''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
app_id = str(app_id)
if request.method == 'HEAD':
if not app_id in monaco.app_ids:
abort(404)
return 'OK'
if request.method == 'GET':
if not app_id in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
app_info = {
'app_id': dbapp.app_id,
'port': dbapp.port,
'nodes': [],
'unused_nodes': [],
}
for k in dbapp.HASH_KEYS:
if hasattr(dbapp, k):
app_info[k] = getattr(dbapp, k)
for node_id, role in dbapp.nodes.iteritems():
node = schema.MonacoNode(node_id=node_id)
node.refresh(r)
app_info['nodes'].append({
'host': node.hostname,
'node_id': node_id,
'role': role
})
app_info['unused_nodes'] = [
node_id for node_id in monaco.node_ids
if not node_id in dbapp.nodes
]
return jsonify(app_info)
if request.method == 'POST':
if app_id in monaco.app_ids:
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
if request.form['name'] != monaco.service_by_app_id[app_id]:
monaco.rename_app(app_id, request.form['name'], r)
job = {
'command': 'update',
'app_id': app_id,
}
for k in schema.App.HASH_KEYS:
if k in request.form:
job[k] = request.form[k]
if 'persist' in job:
job['persist'] = True
else:
job['persist'] = False
if 'slavelb' in job:
job['slavelb'] = True
else:
job['slavelb'] = False
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
else:
# can't create with an app_id pre-specified.
abort(400)
if request.method == 'DELETE':
if not app_id in monaco.app_ids:
abort(404)
dbapp = schema.App(app_id=app_id)
dbapp.refresh(r)
job = {
'command': 'delete',
'app_id': app_id,
}
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
def main():
'''
This is a jazzier version of the node stats reporter.
It will spin up N threads (where N = the number of app Masters on this node)
Those threads will report stats on the config interval
'''
r = redis.StrictRedis(port=config.config['mgmt_port'])
monaco = schema.Monaco()
monaco.refresh(r)
host = config.config['hostname']
node_id = monaco.node_ids_by_hostname[host]
node = schema.MonacoNode(node_id=node_id)
monaco_handler = MonacoHandler(node_id)
monaco_handler.start()
app_threadmap = {}
twem_threadmap = {}
while True:
try:
node.refresh(r)
# Set up this node's master DB handlers
for app_id in app_threadmap.keys():
if app_id not in node.apps:
# child thread should die a natural, painless death
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
for app_id in node.apps:
app = schema.App(app_id=app_id)
app.refresh(r)
if app.nodes[node.node_id] != 'master':
if app_id in app_threadmap:
app_threadmap[app_id].stop()
del app_threadmap[app_id]
STATLOGGER.debug('deleted %s', app_id)
continue
if not app_id in app_threadmap:
# perhaps a new thing
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.debug('started %s', app_id)
elif not app_threadmap[app_id].is_alive():
del app_threadmap[app_id]
app_threadmap[app_id] = AppHandler(app_id, node_id)
app_threadmap[app_id].start()
STATLOGGER.info('restarted %s', app_id)
# Set up this node's twem handlers
for twem_id in twem_threadmap.keys():
if twem_id not in node.twems:
# child thread should die a natural, painless death
twem_threadmap[twem_id].stop()
del twem_threadmap[twem_id]
STATLOGGER.debug('deleted %s', twem_id)
for twem_id in node.twems:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
if not twem_id in twem_threadmap:
# perhaps a new thing
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.debug('started %s', twem_id)
elif not twem_threadmap[twem_id].is_alive():
del twem_threadmap[twem_id]
twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host)
twem_threadmap[twem_id].start()
STATLOGGER.info('restarted %s', twem_id)
except redis.RedisError:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception, exc:
STATLOGGER.exception(exc)
time.sleep(5)
def proxy_api(twem_id):
'''
Twemproxy API:
GET:
200 - gets info about twem in json format
404 - twem_id does not exist
403 - you aint allowed
HEAD:
200 - twem_id exists
404 - twem_id does not exist
POST:
200 - sent update command to master
400 - twem_id does not exist
403 - you aint allowed
DELETE:
200 - sent delete command to master
404 - twem_id does not exist
403 - you aint allowed
'''
r = rediscli()
job_queue = schema.MonacoJobQueue(r)
monaco = schema.Monaco()
monaco.refresh(r)
twem_id = str(twem_id)
if request.method == 'HEAD':
if not twem_id in monaco.twem_ids:
abort(404)
return 'OK'
if request.method == 'GET':
if not twem_id in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
data = {}
for key in schema.MonacoTwem.HASH_KEYS:
if hasattr(twem, key):
data[key] = getattr(twem, key)
data['servers'] = twem.servers
data['extservers'] = twem.extservers
return jsonify(data)
if request.method == 'POST':
if twem_id in monaco.twem_ids:
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
job = {
'command': 'update_proxy',
'twem_id': twem_id,
}
for key in schema.MonacoTwem.HASH_KEYS:
if key in request.form:
job[key] = request.form[key]
if 'servers' in request.form:
job['servers'] = []
for app_id in request.values.getlist('servers'):
if app_id in monaco.app_ids:
job['servers'].append(app_id)
else:
abort(400)
if 'extservers' in request.form:
job['extservers'] = []
ipport = re.compile(
'^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]{4,5})$'
)
serverport = re.compile('^([a-zA-Z0-9-_]*):([0-9]{4,5})$')
for extserver in request.values.getlist('extservers'):
if ipport.match(extserver):
job['extservers'].append(extserver)
elif serverport.match(extserver):
job['extservers'].append(extserver)
else:
abort(400)
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
else:
# can't create. use POST:/api/proxy
abort(400)
if request.method == 'DELETE':
if not twem_id in monaco.twem_ids:
abort(404)
twem = schema.MonacoTwem(twem_id=twem_id)
twem.refresh(r)
job = {
'command': 'delete_proxy',
'twem_id': twem_id,
}
jid = job_queue.pushback_job(job)
return jsonify(jid=jid)
return 'OK'
def list_twems():
''' returns all twem_ids '''
r = rediscli()
monaco = schema.Monaco()
monaco.refresh(r)
return monaco.twem_ids
