def requests(which, country): print "Fetching %s requests for country '%s'..." % (which, country) ret = redis_shell.hgetall("%s:%s-requests" % (country, which)) print "...done fetching %s requests." % which print ret = {k: int(v) for k, v in ret.iteritems()} return ret
def requests(which): print "Fetching %s requests..." ret = redis_shell.hgetall(which + '-requests') print "...done fetching %s requests." print ret = {k: int(v) for k, v in ret.iteritems()} return ret
def run_all_checks(): cfgbysrv = redis_shell.hgetall("srv->cfg") errors = configs_start_with_newline(cfgbysrv) regions = redis_shell.smembers("user-regions") for region in regions: errors.extend(srvs_in_cfgbysrv(region, cfgbysrv)) for region in regions: errors.extend(check_srvq_size(region)) report(errors)
def run_all_checks(): cfgbysrv = redis_shell.hgetall('srv->cfg') errors = configs_start_with_newline(cfgbysrv) regions = redis_shell.smembers('user-regions') for region in regions: errors.extend(srvs_in_cfgbysrv(region, cfgbysrv)) for region in regions: errors.extend(check_srvq_size(region)) report(errors)
def srv_cfg_by_ip(): ret = {} for srv, cfg in redis_shell.hgetall('srv->cfg').iteritems(): ip = yaml.load(cfg).values()[0]['addr'].split(':')[0] if ip in ret: ret[ip][1].append(srv) else: ret[ip] = cfg, [srv] return ret
def fix_live_servers(): srv2cfg = redis_shell.hgetall('srv->cfg') fixed = {} for srv, cfg in srv2cfg.iteritems(): if '\\' in cfg: name, access_data = yaml.load(cfg).items()[0] print "fixing srv %s..." % name fixed_access_data = fix_cert_newlines(access_data) fixed_cfg = serialize_access_data(fixed_access_data, name) fixed[srv] = fixed_cfg redis_shell.hmset('srv->cfg', fixed)
def fix_live_servers(fix_fn, DEBUG_table_name='srv->cfg'): print "fixing live servers..." srv2cfg = redis_shell.hgetall(DEBUG_table_name) fixed = {} for srv, cfg in srv2cfg.iteritems(): needs_fixing, good_cfg = fix_fn(cfg) if needs_fixing: print "proxy with srvid %s needs fixing" % srv fixed[srv] = good_cfg if fixed: print "Fixing %s live proxies..." % len(fixed) redis_shell.hmset(DEBUG_table_name, fixed) print "Done fixing live proxies." else: print "No live proxies to fix!"
def srv2cfg_consistent_with_vps_list(srv2cfg=None, cache=None): if cache is None: cache = model.make_cache() # XXX contrived because of legacy reasons; I'll clean it up soon(TM)... cache.srv2cfg = cache.srv2cfg or srv2cfg or redis_shell.hgetall('srv->cfg') cache.all_vpss = cache.all_vpss or vps_util.all_vpss() errors = model.check_srv2cfg(cache=cache) for err_type, err_cases in errors: if err_type == 'bad IP in srv->cfg': # Delete entries pointing to proxies that are no longer ours. names = redis_shell.hmget('srv->name', err_cases.keys()) for name, (srv, ip) in zip(names, err_cases.iteritems()): try: print "Retiring", name, ip, srv vps_util.actually_retire_proxy(name=name, ip=ip, srv=srv) except: alert_exception("retire non-existing proxy %s (%s), srvid %s" % (name, ip, srv)) return errors
def actually_offload_proxy(name=None, ip=None, srv=None, pipeline=None): name, ip, srv = nameipsrv(name, ip, srv) region = region_by_name(name) client_table_key = region + ':clientip->srv' packed_srv = redis_util.pack_srv(srv) #XXX: a proxy -> {clients} index is sorely needed! # Getting the set of clients assigned to this proxy takes a long time # currently. Let's get it done before pulling the replacement proxy, # so we're less likely to be left with an empty proxy if interrupted. clients = set(pip for pip, psrv in redis_shell.hgetall(client_table_key).iteritems() if psrv == packed_srv) dest = pull_from_srvq(region) # It's still possible that we'll crash or get rebooted here, so the # destination server will be left empty. The next closed proxy compaction # job will find this proxy and assign some users to it or mark it for # retirement. dest_psrv = redis_util.pack_srv(dest.srv) redis_shell.hmset(client_table_key, {pip: dest_psrv for pip in clients}) print "Offloaded clients from %s (%s) to %s (%s)" % (name, ip, dest.name, dest.ip)
def retire_lcs(name, ip, cfgcache=util.Cache(timeout=60*60, update_fn=lambda: redis_shell.hgetall('cfgbysrv'))): if name.startswith('fp-jp-'): dc = 'vltok1' elif name.startswith('fp-nl-'): dc = 'doams3' else: assert False srvs = [srv for srv, cfg in cfgcache.get().iteritems() if yaml.load(cfg).values()[0]['addr'].split(':')[0] == ip] if srvs: redis_shell.hdel('cfgbysrv', *srvs) redis_shell.incr('srvcount') else: "No configs left to delete for %s." % name redis_shell.lrem(dc + ':vpss', name) redis_shell.incr(dc + ':vpss:version')
def actually_offload_proxy(name=None, ip=None, srv=None, pipeline=None): name, ip, srv = nameipsrv(name, ip, srv) region = region_by_name(name) client_table_key = region + ':clientip->srv' packed_srv = redis_util.pack_srv(srv) #XXX: a proxy -> {clients} index is sorely needed! # Getting the set of clients assigned to this proxy takes a long time # currently. Let's get it done before pulling the replacement proxy, # so we're less likely to be left with an empty proxy if interrupted. clients = set( pip for pip, psrv in redis_shell.hgetall(client_table_key).iteritems() if psrv == packed_srv) dest = pull_from_srvq(region) # It's still possible that we'll crash or get rebooted here, so the # destination server will be left empty. The next closed proxy compaction # job will find this proxy and assign some users to it or mark it for # retirement. dest_psrv = redis_util.pack_srv(dest.srv) redis_shell.hmset(client_table_key, {pip: dest_psrv for pip in clients}) print "Offloaded clients from %s (%s) to %s (%s)" % (name, ip, dest.name, dest.ip)
def run_all_checks(): print "Fetching config data..." srv2cfg = redis_shell.hgetall('srv->cfg') print "Performing checks..." cache = model.make_cache() # This is new code, so let's test it in a cushion to start with. try: print "Checking that srv->cfg table is consistent with the VPS listing..." errors = srv2cfg_consistent_with_vps_list(srv2cfg, cache) except: alert_exception("trying to check consistency between srv->cfg and all_vpss") errors = [] try: print "Check that we don't have duplicate names" errors.extend(no_duplicate_names(cache)) except: alert_exception("trying to check for duplicate VPS names") print "Checking that configs start with a newline..." errors.extend(configs_start_with_newline(srv2cfg)) regions = redis_shell.smembers('user-regions') print "Checking that slice server entries are in srv->cfg..." for region in regions: print " (region %s)..." % region errors.extend(slice_srvs_in_srv2cfg(region, srv2cfg)) print "Checking server queue size..." for region in regions: print " (region %s)..." % region errors.extend(srvq_size(region)) print "Checking server queue integrity..." for region in regions: print " (region %s)..." % region try: errors.extend(srvq_integrity(region, cache=cache)) except: alert_exception("trying to check server queue integrity") print "Check that regional fallbacks and honeypots are in srv->cfg..." for region in regions: print " (region %s)..." % region errors.extend(fallbacks_and_honeypots_in_srv_table(region, srv2cfg)) report(errors)
import yaml from alert import alert from redis_util import redis_shell prefix = 'fallbacks-to-check' try: local_version = file(prefix + '-version').read() except IOError: local_version = None remote_version = redis_shell.get('srvcount') if local_version != remote_version: suppress = redis_shell.smembers('checkfallbacks-suppress') json.dump([yaml.load(cfg).values()[0] for srv, cfg in redis_shell.hgetall('srv->cfg').iteritems() if srv not in suppress], file(prefix + '.json', 'w')) file(prefix + '-version', 'w').write(remote_version) cmd = subprocess.Popen("checkfallbacks -fallbacks %s.json -connections 20 | grep '\[failed fallback check\]'" % prefix, shell=True, stdout=subprocess.PIPE) errors = list(cmd.stdout) if errors: for error in errors: print error alert(type='checkfallbacks-failures', details={'errors': errors}, title='Proxy check failures', text="".join(error[len('[failed fallback check] '):] + "\n"