def new_proxy_api(): ''' for creating new proxies ''' r = rediscli() job_queue = schema.MonacoJobQueue(r) monaco = schema.Monaco() monaco.refresh(r) if request.form['name'] in monaco.twem_ids_by_name: abort(400) job = { 'command': 'new_proxy', } for k in schema.MonacoTwem.HASH_KEYS: if k in request.form: job[k] = request.form[k] job['servers'] = [] for app_id in request.values.getlist('servers'): if app_id in monaco.app_ids: job['servers'].append(app_id) else: abort(400) job['extservers'] = [] ipport = re.compile( '^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]{4,5})$') serverport = re.compile('^([a-zA-Z0-9_-]*):([0-9]{4,5})$') for extserver in request.values.getlist('extservers'): if ipport.match(extserver): job['extservers'].append(extserver) elif serverport.match(extserver): job['extservers'].append(extserver) else: abort(400) job['hash'] = 'murmur' jid = job_queue.pushback_job(job) return jsonify(jid=jid)
def __init__(self): monaco = schema.Monaco() self.r = redis.StrictRedis(port=config['mgmt_port']) # terniaries are always a bad idea. this is a mess of exceptions waiting to cascade so FIXME if self.r.info()['role'] == 'master': self.rmaster = redis.StrictRedis(port=config['mgmt_port']) else: self.rmaster = redis.StrictRedis(host=self.r.info()['master_host'], port=config['mgmt_port'], socket_connect_timeout=1, socket_timeout=1) monaco.refresh(self.r) node_id = monaco.node_ids_by_hostname[config['hostname']] self.node = schema.MonacoNode(node_id=node_id) self.health_data = {} # dictionary of app_id -> DB health self.app_clients = {} # dictionary of app_id -> redis clients self.rps = redis.StrictRedis(port=config['mgmt_port']) self.pubsub = self.rps.pubsub(ignore_subscribe_messages=True) self.lock = threading.Lock() self._subscriptions = {} self.logger = logging.getLogger('monaco.slave') self.redmanager = RedisMgmt() self.nutmanager = NutMgmt() # for slave based health-checks self.sched = Scheduler(daemon=True) self.sched.start() self.sched.add_interval_job(self.node_health, seconds=5) # TODO: Tune self.health_check_pool = ThreadPool(10) atexit.register(lambda: self.sched.shutdown(wait=False))
def list_proxies_api(): ''' lists twems''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if 'owner' in request.args: twems = list_twems_by_owner(request['owner']) elif 'operator' in request.args: twems = list_twems_by_operator(set(request['operator'])) else: twems = list_twems() twem_data = {} for twem_id in twems: try: twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) twem_data[twem_id] = { 'name': twem.name, 'lb': 'tcp://%s:%s' % (app.config['MONACO_DB'], twem_id), 'servers': twem.servers, } except Exception: twem_data[twem_id] = {'service': monaco.name_by_twem_id[twem_id]} return jsonify(twem_data)
def check_clusters_vs_node_apps(testlogger, r): ''' Check that clusters match node-app sets ''' result = True m = schema.Monaco() m.refresh(r) for app_id in m.app_ids: app = schema.App(app_id=app_id) app.refresh(r) for node_id, _ in app.nodes.iteritems(): if not app_id in r.smembers(schema.NODE_APPS_TMPL % node_id): testlogger.error( "App %s has node %s in it's cluster, but the node doesn't have it in it's app-set", app_id, node_id) result = False for node_id in m.node_ids: node = schema.MonacoNode(node_id=node_id) node.refresh(r) for app_id in node.apps: if not node_id in r.hkeys(schema.APP_CLUSTER_TMPL % app_id): testlogger.error( "Node %s has app %s in its app-set, but the corresponding app doesn't have the node in it's cluster", node_id, app_id) result = False return result
def new_app_api(): ''' for creating new apps ''' r = rediscli() job_queue = schema.MonacoJobQueue(r) monaco = schema.Monaco() monaco.refresh(r) if request.form['name'] in monaco.app_ids_by_service: abort(400) job = { 'command': 'new', } for k in schema.App.HASH_KEYS: if k in request.form: job[k] = request.form[k] if not 'persist' in request.form: job['persist'] = False else: job['persist'] = True if not 'slavelb' in request.form: job['slavelb'] = False else: job['slavelb'] = True jid = job_queue.pushback_job(job) if not jid: # just retry in app abort(503) return jsonify(jid=jid)
def check_apps_and_clusters(testlogger, r): ''' Check that apps:clusters is 1:1 ''' result = True app_hashes = [ key for key in r.keys(schema.APP_HASH_TMPL % '*') if not (key.endswith('cluster') or key.endswith('status') or key.endswith('version')) ] app_clusters = r.keys(schema.APP_CLUSTER_TMPL % '*') m = schema.Monaco() m.refresh(r) for app_id in m.app_ids: hashkey = schema.APP_HASH_TMPL % app_id clusterkey = schema.APP_CLUSTER_TMPL % app_id if hashkey in app_hashes: app_hashes.remove(hashkey) else: testlogger.error("App %s doesn't have a corresponding app hash", app_id) result = False if clusterkey in app_clusters: app_clusters.remove(clusterkey) else: testlogger.error( "App %s doesn't have a corresponding cluster hash", app_id) result = False if len(app_hashes) != 0: testlogger.warn('Extra app hashes: %s', app_hashes) result = False if len(app_clusters) != 0: testlogger.warn('Extra cluster hashes: %s', app_clusters) result = False return result
def app_redis_api(app_id): ''' Simple REST api to redis DBs. GET, PUT, DELETE are key operations, and POST allows for any command method = 'GET': ../redis?key=key return r.get('key') method = 'PUT': ../redis?key=key&val=val return r.set('key', 'val') method = 'DELETE': ../redis?key=key return r.delete('key') method = 'POST': ../redis?cmd=hset&args=key,hashkey,hashval Request args return getattr(r,cmd)(*args.split(',')) aka r.hset(key, hashkey, hashval) ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) app_id = str(app_id) if not app_id in monaco.app_ids: abort(404) userapp = schema.App(app_id=app_id) userapp.refresh(r) master_host = None for node_id, role in userapp.nodes.iteritems(): if role == 'master': master_host = monaco.hostnames_by_node_id[node_id] break assert master_host != None r = StrictRedis(master_host, userapp.port) if request.method == 'GET': if 'key' not in request.args: abort(400) return r.get(request.args['key']) if request.method == 'PUT': if 'key' not in request.args or 'val' not in request.args: abort(400) return r.set(request.args['key'], request.args['val']) if request.method == 'DELETE': if 'key' not in request.args: abort(400) return r.delete(request.args['key']) if request.method == 'POST': if 'cmd' not in request.args or not hasattr(r, request.args['cmd']): abort(400) if 'args' in request.args: args = request.args['args'].split(',') else: args = [] return getattr(r, request.args['cmd'])(*args) abort(400)
def app_view(app_id): ''' Web UI for an App ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if not str(app_id) in monaco.app_ids: abort(404) dbapp = schema.App(app_id=app_id) dbapp.refresh(r) data = {} for node, role in dbapp.nodes.iteritems(): node = schema.MonacoNode(node_id=node) node.refresh(r) if role == 'master': data[role] = {'host': node.hostname, 'port': dbapp.port} elif role in data: data[role].append({'host': node.hostname, 'port': dbapp.port}) else: data[role] = [{'host': node.hostname, 'port': dbapp.port}] data['app_id'] = app_id data['name'] = dbapp.name # scale bytes to human readable mb/gb data['maxmemory'] = dbapp.maxmemory data['maxmemory_policy'] = dbapp.maxmemory_policy data['persist'] = dbapp.persist == 'True' data['replicas'] = dbapp.replicas data['slavelb'] = dbapp.slavelb == 'True' data['owner'] = dbapp.owner data['operator'] = dbapp.operator data['memory_target'] = '&target=monaco.%s.%s.%s.used_memory' % ( app.config['ENV'], app.config['LOCATION'], app_id, ) data[ 'rps_target'] = '&target=monaco.%s.%s.%s.instantaneous_ops_per_sec' % ( app.config['ENV'], app.config['LOCATION'], app_id, ) data['conn_target'] = '&target=monaco.%s.%s.%s.connected_clients' % ( app.config['ENV'], app.config['LOCATION'], app_id, ) data['cpu_target'] = '&target=monaco.%s.%s.%s.cpu_percent' % ( app.config['ENV'], app.config['LOCATION'], app_id, ) return render_template('db.html', **data)
def list_apps_by_owner(owner): ''' list all apps where owner == app.owner ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) apps = [] for app_id in monaco.app_ids: try: dbapp = schema.App(app_id=app_id) dbapp.refresh(r) if dbapp.owner == owner: apps.append(app_id) except Exception: pass return apps
def list_twems_by_owner(owner): ''' list all twem_ids where owner == twem.owner ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) twems = [] for twem_id in monaco.twem_ids: try: twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) if twem.owner == owner: twems.append(twem_id) except Exception: pass return twems
def list_apps_by_operator(groups): ''' lists all apps operated by one of the groups ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) apps = [] for app_id in monaco.app_ids: try: dbapp = schema.App(app_id=app_id) dbapp.refresh(r) if dbapp.operator in groups: apps.append(app_id) except Exception: pass return apps
def list_twems_by_operator(groups): ''' lists all twems operated by one of the groups ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) twems = [] for twem_id in monaco.twem_ids: try: twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) if twem.operator in groups: twems.append(twem_id) except Exception: pass return twems
def validate_app_invariences(testlogger, r): ''' Ensure that the user specified replica count is maintained ''' result = True m = schema.Monaco() m.refresh(r) for app_id in m.app_ids: app = schema.App(app_id=app_id) app.refresh(r) if int(app.replicas) != len(app.nodes): testlogger.error("App %s doesn't have the desired replica count", app_id) result = False return result
def check_allocated_nodes_online(testlogger, r): ''' Ensure that any node allocated to hold a redis instance for an app is online NOTE: This only verifies against the reported 'status'. ''' result = True m = schema.Monaco() m.refresh(r) for app_id in m.app_ids: for node_id, _ in r.hgetall(schema.APP_CLUSTER_TMPL % app_id).iteritems(): if r.hget(schema.NODE_HASH_TMPL % node_id, 'status') != 'UP': testlogger.error( "App %s has node %s allocated to host an instance, despite the node being marked DOWN." % (app_id, node_id)) result = False return result
def twem_conf_struct(self, twem, retry=True): ''' Given a schema.MonacoTwem, returns the nutcracker config for that proxy in dict form ''' try: if type(twem) != schema.MonacoTwem: twem = schema.MonacoTwem(twem_id=twem) twem.refresh(self.r) conf = {} for key in schema.MonacoTwem.HASH_KEYS: if hasattr(twem, key) and key in self.CONF_KEYS: conf[key] = getattr(twem, key) conf['listen'] = twem.listen conf['auto_eject_hosts'] = twem.auto_eject_hosts conf['redis'] = True conf['servers'] = [] if len(twem.servers) == 1: # configure to proxy across the master and slaves of a single monaco db, using physical hostnames app = schema.App(app_id=twem.servers[0]) app.refresh(self.r) monaco = schema.Monaco() monaco.refresh(self.r) for node_id in app.nodes: node = schema.MonacoNode(node_id=node_id) node.refresh(self.r) conf['servers'].append('%s:%s:1' % (node.FQDN, app.port)) else: # configure to proxy across a set of monaco dbs, using the loadbalanced hostname for app_id in twem.servers: conf['servers'].append( '%s:%s:1' % (config['loadbalancer']['hostname'], app_id)) # Allow for external servers that are manually specified if twem.extservers: for server in twem.extservers: conf['servers'].append('%s:1' % server) return {twem.name: conf} except redis.RedisError, err: self.r = redis.StrictRedis(port=config['mgmt_port']) if retry: return self.twem_conf_struct(twem, retry=False) else: self.logger.exception(err)
def list_nodes_api(): ''' This provides a RESTful endpoint for listing MonacoNodes HEAD: Returns {'node_ids': [node_id, node_id]}, which only queries master mgmt db GET: Returns HEAD + {'<node_id>': {NODE INFO DICT}}, which queries redis servers on each node ''' data = {} r = rediscli() monaco = schema.Monaco() monaco.refresh(r) data['node_ids'] = monaco.node_ids for node_id in data['node_ids']: data[node_id] = {} data[node_id]['hostname'] = monaco.hostnames_by_node_id[node_id] if request.method == 'HEAD': continue try: rtemp = StrictRedis(host=data[node_id]['hostname'], port=app.config['MGMT_PORT'], socket_connect_timeout=1, socket_timeout=0.5) info = rtemp.info() data[node_id]['role'] = info['role'] if data[node_id]['role'] == 'slave': data[node_id]['role_details'] = {'master': info['master_host']} else: data[node_id]['role_details'] = {} data[node_id]['role_details']['connected_slaves'] = info[ 'connected_slaves'] for idx in xrange(int(info['connected_slaves'])): data[node_id]['role_details']['slave%d' % idx] = info['slave%d' % idx] node = schema.MonacoNode(node_id=node_id) node.refresh(r) data[node_id]['mem_usage'] = '%sM' % node.app_info(r)['memory'] data[node_id]['up'] = True except Exception: data[node_id]['up'] = False data[node_id]['role'] = None data[node_id]['role_details'] = None data[node_id]['mem_usage'] = None data[node_id]['net_usage'] = None return jsonify(data)
def node_view(node_id): ''' Web view for MonacoNode ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) node_id = str(node_id) if not node_id in monaco.node_ids: abort(404) node = schema.MonacoNode(node_id=node_id) node.refresh(r) appinfo = node.app_info(r) data = { 'node_id': node_id, 'hostname': node.hostname, 'FQDN': node.FQDN, 'total_memory': node.total_memory, 'rack': node.rack, 'status': node.status, 'used_memory': appinfo['memory'], 'memory_percent': round(100.0 * appinfo['memory'] / (int(node.total_memory) / 2.0), 2), 'master_servers': len(appinfo['masters']), 'masters': map(str, sorted(map(int, appinfo['masters']))), 'slave_servers': len(appinfo['slaves']), 'slaves': map(str, sorted(map(int, appinfo['slaves']))), 'twemproxy_servers': len(node.twems), # General info 'nodes': map(str, sorted(map(int, monaco.node_ids))), } return render_template("node.html", **data)
def proxy_stats(twem_id): ''' Returns live aggregates for a given proxy ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if not str(twem_id) in monaco.twem_ids: abort(404) twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) aggregate_rps = 0 aggregate_connections = 0 if len(twem.servers) == 1: dbapp = schema.App(app_id=twem.servers[0]) dbapp.refresh(r) for node_id, _ in dbapp.nodes.iteritems(): appcli = StrictRedis(monaco.hostnames_by_node_id[node_id], dbapp.port) info = appcli.info() if 'instantaneous_ops_per_sec' in info: aggregate_rps += info['instantaneous_ops_per_sec'] if 'connected_clients' in info: aggregate_connections += info['connected_clients'] else: for app_id in twem.servers: dbapp = schema.App(app_id=app_id) dbapp.refresh(r) appcli = dbapp.get_master_connection(r) info = appcli.info() if 'instantaneous_ops_per_sec' in info: aggregate_rps += info['instantaneous_ops_per_sec'] if 'connected_clients' in info: aggregate_connections += info['connected_clients'] return jsonify({ 'total_rps': aggregate_rps, 'total_connections': aggregate_connections })
def new_node_api(): ''' Create new nodes by POST'ing info to this endpoint. Get a listing of nodes ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if request.method == 'POST': if str(request.form['node_id']) in monaco.node_ids: abort(400) newnode = schema.MonacoNode(node_id=request.form['node_id']) newnode.apps = [] newnode.twems = [] newnode.hostname = request.form['hostname'] newnode.FQDN = request.form['FQDN'] newnode.total_memory = request.form['total_memory'] newnode.rack = request.form['rack'] newnode.write(r) monaco.new_node(newnode, r) return 'OK'
def list_app_api(): ''' lists apps''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if 'owner' in request.args: apps = list_apps_by_owner(request['owner']) elif 'operator' in request.args: apps = list_apps_by_operator(set(request['operator'])) else: apps = list_apps() app_data = {} for app_id in apps: try: dbapp = schema.App(app_id=app_id) dbapp.refresh(r) masternode = schema.MonacoNode(node_id=dbapp.master) masternode.refresh(r) mastercli = dbapp.get_master_connection(r) info = mastercli.info() used = float(info['used_memory']) / (1024 * 1024) total = int(dbapp.maxmemory) // (1024 * 1024) percent = round((100 * used) / total, 2) used = round(used, 2) app_data[app_id] = { 'service': dbapp.name, 'exposure': 'tcp://%s:%s' % (masternode.FQDN, dbapp.port), 'memory_used': used, 'memory_total': total, 'memory_percent': percent, 'connected_clients': info['connected_clients'], 'rps': info['instantaneous_ops_per_sec'], } except Exception: app_data[app_id] = {'service': monaco.service_by_app_id[app_id]} return jsonify(app_data)
def node_api(node_id): ''' This provides a RESTful endpoint for MonacoNode management GET- get MonacoNode info POST- with updated info DELETE- an existing node to remove from distribution ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if request.method == 'GET': if not str(node_id) in monaco.node_ids: abort(404) node = schema.MonacoNode(node_id=str(node_id)) node.refresh(r) data = dict(node.__dict__.items() + node.app_info(r).items()) return jsonify(data) if request.method == 'POST': if not str(node_id) in monaco.node_ids: abort(404) node = schema.MonacoNode(node_id=node_id) node.refresh(r) node.total_memory = request.form['total_memory'] node.rack = request.form['rack'] node.write(r) return 'OK' if request.method == 'DELETE': if str(node_id) in monaco.node_ids: abort(400) nodetokill = schema.MonacoNode(node_id=node_id) if len(nodetokill.apps) != 0 or nodetokill.status != 'MAINTENANCE': abort(400) monaco.delete_node(nodetokill, r) nodetokill.delete(r) return 'OK'
def proxy_view(twem_id): ''' Templates the proxy view ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) if not str(twem_id) in monaco.twem_ids: abort(404) twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) data = {} data['twem_id'] = twem_id data['name'] = twem.name data['servers'] = twem.servers data['extservers'] = twem.extservers data['dbinfo'] = {} for app_id in twem.servers: # Get usage info on all backend DBs dbapp = schema.App(app_id=app_id) dbapp.refresh(r) mastercli = dbapp.get_master_connection(r) info = mastercli.info() used = float(info['used_memory']) / (1024 * 1024) total = int(dbapp.maxmemory) // (1024 * 1024) percent = round((100 * used) / total, 2) used = round(used, 2) data['dbinfo'][app_id] = {} data['dbinfo'][app_id]['total'] = total data['dbinfo'][app_id]['used'] = used data['dbinfo'][app_id]['percent'] = percent data['distribution'] = twem.distribution data['owner'] = twem.owner data['operator'] = twem.operator # choices for servers data['all_servers'] = [app_id for app_id in list_apps()] return render_template('proxy.html', **data)
def check_all_nodes_in_nodes_list(testlogger, r): ''' Same as above, but for nodes ''' m = schema.Monaco() m.refresh(r) getid = re.compile(r'-([0-9]*)$') noextra = True for node in r.keys(schema.NODE_HASH_TMPL % '*'): match = getid.search(node) if not match: continue node = match.group(1) if node in m.node_ids: m.node_ids.remove(node) else: testlogger.warn('Extra node struct found at key: %s' % node) noextra = False for node in m.node_ids: testlogger.error( 'Node %s found in top-level node list, but no corresponding struct found' % node) return len(m.node_ids) == 0 and noextra
def check_cluster_sanity(testlogger, r): ''' Check that each cluster has only 1 master ''' result = True m = schema.Monaco() m.refresh(r) for app_id in m.app_ids: cluster = r.hgetall(schema.APP_CLUSTER_TMPL % app_id) master = [k for k, v in cluster.iteritems() if v == 'master'] if len(master) != 1: testlogger.error('App %s has %s masters!' % (app_id, len(master))) result = False invalid = [ k for k, v in cluster.iteritems() if not v in ['master', 'slave', 'slave-write'] ] if len(invalid) > 0: testlogger.error('App %s has an invalid cluster specification: %s', app_id, cluster) result = False return result
def check_all_apps_in_apps_list(testlogger, r): ''' Check if the monaco-apps set represents the set of monaco app hashes in r ''' m = schema.Monaco() m.refresh(r) getid = re.compile(r'-([0-9]*)$') noextra = True # mix the template with a wildcard to match all for app in r.keys(schema.APP_HASH_TMPL % '*'): match = getid.search(app) if not match: continue app = match.group(1) if app in m.app_ids: m.app_ids.remove(app) else: testlogger.warn('Extra app found at key: %s' % app) noextra = False for app in m.app_ids: testlogger.error( 'App %s found in top-level app list, but no corresponding struct found' % app) return len(m.app_ids) == 0 and noextra
def list_apps(): ''' returns all app_ids ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) return monaco.app_ids
def app_api(app_id): ''' App API: GET: 200 - gets info about app in json format 404 - app_id does not exist 403 - you aint allowed HEAD: 200 - app_id exists 404 - app_id does not exist POST: 200 - sent update command to master 400 - app_id does not exist 403 - you aint allowed DELETE: 200 - sent delete command to master 404 - app_id does not exist 403 - you aint allowed ''' r = rediscli() job_queue = schema.MonacoJobQueue(r) monaco = schema.Monaco() monaco.refresh(r) app_id = str(app_id) if request.method == 'HEAD': if not app_id in monaco.app_ids: abort(404) return 'OK' if request.method == 'GET': if not app_id in monaco.app_ids: abort(404) dbapp = schema.App(app_id=app_id) dbapp.refresh(r) app_info = { 'app_id': dbapp.app_id, 'port': dbapp.port, 'nodes': [], 'unused_nodes': [], } for k in dbapp.HASH_KEYS: if hasattr(dbapp, k): app_info[k] = getattr(dbapp, k) for node_id, role in dbapp.nodes.iteritems(): node = schema.MonacoNode(node_id=node_id) node.refresh(r) app_info['nodes'].append({ 'host': node.hostname, 'node_id': node_id, 'role': role }) app_info['unused_nodes'] = [ node_id for node_id in monaco.node_ids if not node_id in dbapp.nodes ] return jsonify(app_info) if request.method == 'POST': if app_id in monaco.app_ids: dbapp = schema.App(app_id=app_id) dbapp.refresh(r) if request.form['name'] != monaco.service_by_app_id[app_id]: monaco.rename_app(app_id, request.form['name'], r) job = { 'command': 'update', 'app_id': app_id, } for k in schema.App.HASH_KEYS: if k in request.form: job[k] = request.form[k] if 'persist' in job: job['persist'] = True else: job['persist'] = False if 'slavelb' in job: job['slavelb'] = True else: job['slavelb'] = False jid = job_queue.pushback_job(job) return jsonify(jid=jid) else: # can't create with an app_id pre-specified. abort(400) if request.method == 'DELETE': if not app_id in monaco.app_ids: abort(404) dbapp = schema.App(app_id=app_id) dbapp.refresh(r) job = { 'command': 'delete', 'app_id': app_id, } jid = job_queue.pushback_job(job) return jsonify(jid=jid)
def main(): ''' This is a jazzier version of the node stats reporter. It will spin up N threads (where N = the number of app Masters on this node) Those threads will report stats on the config interval ''' r = redis.StrictRedis(port=config.config['mgmt_port']) monaco = schema.Monaco() monaco.refresh(r) host = config.config['hostname'] node_id = monaco.node_ids_by_hostname[host] node = schema.MonacoNode(node_id=node_id) monaco_handler = MonacoHandler(node_id) monaco_handler.start() app_threadmap = {} twem_threadmap = {} while True: try: node.refresh(r) # Set up this node's master DB handlers for app_id in app_threadmap.keys(): if app_id not in node.apps: # child thread should die a natural, painless death app_threadmap[app_id].stop() del app_threadmap[app_id] STATLOGGER.debug('deleted %s', app_id) for app_id in node.apps: app = schema.App(app_id=app_id) app.refresh(r) if app.nodes[node.node_id] != 'master': if app_id in app_threadmap: app_threadmap[app_id].stop() del app_threadmap[app_id] STATLOGGER.debug('deleted %s', app_id) continue if not app_id in app_threadmap: # perhaps a new thing app_threadmap[app_id] = AppHandler(app_id, node_id) app_threadmap[app_id].start() STATLOGGER.debug('started %s', app_id) elif not app_threadmap[app_id].is_alive(): del app_threadmap[app_id] app_threadmap[app_id] = AppHandler(app_id, node_id) app_threadmap[app_id].start() STATLOGGER.info('restarted %s', app_id) # Set up this node's twem handlers for twem_id in twem_threadmap.keys(): if twem_id not in node.twems: # child thread should die a natural, painless death twem_threadmap[twem_id].stop() del twem_threadmap[twem_id] STATLOGGER.debug('deleted %s', twem_id) for twem_id in node.twems: twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) if not twem_id in twem_threadmap: # perhaps a new thing twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host) twem_threadmap[twem_id].start() STATLOGGER.debug('started %s', twem_id) elif not twem_threadmap[twem_id].is_alive(): del twem_threadmap[twem_id] twem_threadmap[twem_id] = TwemHandler(twem_id, node_id, host) twem_threadmap[twem_id].start() STATLOGGER.info('restarted %s', twem_id) except redis.RedisError: r = redis.StrictRedis(port=config.config['mgmt_port']) except Exception, exc: STATLOGGER.exception(exc) time.sleep(5)
def proxy_api(twem_id): ''' Twemproxy API: GET: 200 - gets info about twem in json format 404 - twem_id does not exist 403 - you aint allowed HEAD: 200 - twem_id exists 404 - twem_id does not exist POST: 200 - sent update command to master 400 - twem_id does not exist 403 - you aint allowed DELETE: 200 - sent delete command to master 404 - twem_id does not exist 403 - you aint allowed ''' r = rediscli() job_queue = schema.MonacoJobQueue(r) monaco = schema.Monaco() monaco.refresh(r) twem_id = str(twem_id) if request.method == 'HEAD': if not twem_id in monaco.twem_ids: abort(404) return 'OK' if request.method == 'GET': if not twem_id in monaco.twem_ids: abort(404) twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) data = {} for key in schema.MonacoTwem.HASH_KEYS: if hasattr(twem, key): data[key] = getattr(twem, key) data['servers'] = twem.servers data['extservers'] = twem.extservers return jsonify(data) if request.method == 'POST': if twem_id in monaco.twem_ids: twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) job = { 'command': 'update_proxy', 'twem_id': twem_id, } for key in schema.MonacoTwem.HASH_KEYS: if key in request.form: job[key] = request.form[key] if 'servers' in request.form: job['servers'] = [] for app_id in request.values.getlist('servers'): if app_id in monaco.app_ids: job['servers'].append(app_id) else: abort(400) if 'extservers' in request.form: job['extservers'] = [] ipport = re.compile( '^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]{4,5})$' ) serverport = re.compile('^([a-zA-Z0-9-_]*):([0-9]{4,5})$') for extserver in request.values.getlist('extservers'): if ipport.match(extserver): job['extservers'].append(extserver) elif serverport.match(extserver): job['extservers'].append(extserver) else: abort(400) jid = job_queue.pushback_job(job) return jsonify(jid=jid) else: # can't create. use POST:/api/proxy abort(400) if request.method == 'DELETE': if not twem_id in monaco.twem_ids: abort(404) twem = schema.MonacoTwem(twem_id=twem_id) twem.refresh(r) job = { 'command': 'delete_proxy', 'twem_id': twem_id, } jid = job_queue.pushback_job(job) return jsonify(jid=jid) return 'OK'
def list_twems(): ''' returns all twem_ids ''' r = rediscli() monaco = schema.Monaco() monaco.refresh(r) return monaco.twem_ids