def requests(which, country):
print "Fetching %s requests for country '%s'..." % (which, country)
ret = redis_shell.hgetall("%s:%s-requests" % (country, which))
print "...done fetching %s requests." % which
print
ret = {k: int(v) for k, v in ret.iteritems()}
return ret
def requests(which):
print "Fetching %s requests..."
ret = redis_shell.hgetall(which + '-requests')
print "...done fetching %s requests."
print
ret = {k: int(v) for k, v in ret.iteritems()}
return ret
def requests(which):
print "Fetching %s requests..."
ret = redis_shell.hgetall(which + '-requests')
print "...done fetching %s requests."
print
ret = {k: int(v) for k, v in ret.iteritems()}
return ret
def requests(which, country):
print "Fetching %s requests for country '%s'..." % (which, country)
ret = redis_shell.hgetall("%s:%s-requests" % (country, which))
print "...done fetching %s requests." % which
print
ret = {k: int(v) for k, v in ret.iteritems()}
return ret
def run_all_checks():
cfgbysrv = redis_shell.hgetall("srv->cfg")
errors = configs_start_with_newline(cfgbysrv)
regions = redis_shell.smembers("user-regions")
for region in regions:
errors.extend(srvs_in_cfgbysrv(region, cfgbysrv))
for region in regions:
errors.extend(check_srvq_size(region))
report(errors)
def run_all_checks():
cfgbysrv = redis_shell.hgetall('srv->cfg')
errors = configs_start_with_newline(cfgbysrv)
regions = redis_shell.smembers('user-regions')
for region in regions:
errors.extend(srvs_in_cfgbysrv(region, cfgbysrv))
for region in regions:
errors.extend(check_srvq_size(region))
report(errors)
def srv_cfg_by_ip():
ret = {}
for srv, cfg in redis_shell.hgetall('srv->cfg').iteritems():
ip = yaml.load(cfg).values()[0]['addr'].split(':')[0]
if ip in ret:
ret[ip][1].append(srv)
else:
ret[ip] = cfg, [srv]
return ret
def srv_cfg_by_ip():
ret = {}
for srv, cfg in redis_shell.hgetall('srv->cfg').iteritems():
ip = yaml.load(cfg).values()[0]['addr'].split(':')[0]
if ip in ret:
ret[ip][1].append(srv)
else:
ret[ip] = cfg, [srv]
return ret
def fix_live_servers():
srv2cfg = redis_shell.hgetall('srv->cfg')
fixed = {}
for srv, cfg in srv2cfg.iteritems():
if '\\' in cfg:
name, access_data = yaml.load(cfg).items()[0]
print "fixing srv %s..." % name
fixed_access_data = fix_cert_newlines(access_data)
fixed_cfg = serialize_access_data(fixed_access_data, name)
fixed[srv] = fixed_cfg
redis_shell.hmset('srv->cfg', fixed)
def fix_live_servers():
srv2cfg = redis_shell.hgetall('srv->cfg')
fixed = {}
for srv, cfg in srv2cfg.iteritems():
if '\\' in cfg:
name, access_data = yaml.load(cfg).items()[0]
print "fixing srv %s..." % name
fixed_access_data = fix_cert_newlines(access_data)
fixed_cfg = serialize_access_data(fixed_access_data, name)
fixed[srv] = fixed_cfg
redis_shell.hmset('srv->cfg', fixed)
def fix_live_servers(fix_fn, DEBUG_table_name='srv->cfg'):
print "fixing live servers..."
srv2cfg = redis_shell.hgetall(DEBUG_table_name)
fixed = {}
for srv, cfg in srv2cfg.iteritems():
needs_fixing, good_cfg = fix_fn(cfg)
if needs_fixing:
print "proxy with srvid %s needs fixing" % srv
fixed[srv] = good_cfg
if fixed:
print "Fixing %s live proxies..." % len(fixed)
redis_shell.hmset(DEBUG_table_name, fixed)
print "Done fixing live proxies."
else:
print "No live proxies to fix!"
def fix_live_servers(fix_fn, DEBUG_table_name='srv->cfg'):
print "fixing live servers..."
srv2cfg = redis_shell.hgetall(DEBUG_table_name)
fixed = {}
for srv, cfg in srv2cfg.iteritems():
needs_fixing, good_cfg = fix_fn(cfg)
if needs_fixing:
print "proxy with srvid %s needs fixing" % srv
fixed[srv] = good_cfg
if fixed:
print "Fixing %s live proxies..." % len(fixed)
redis_shell.hmset(DEBUG_table_name, fixed)
print "Done fixing live proxies."
else:
print "No live proxies to fix!"
def srv2cfg_consistent_with_vps_list(srv2cfg=None, cache=None):
if cache is None:
cache = model.make_cache()
# XXX contrived because of legacy reasons; I'll clean it up soon(TM)...
cache.srv2cfg = cache.srv2cfg or srv2cfg or redis_shell.hgetall('srv->cfg')
cache.all_vpss = cache.all_vpss or vps_util.all_vpss()
errors = model.check_srv2cfg(cache=cache)
for err_type, err_cases in errors:
if err_type == 'bad IP in srv->cfg':
# Delete entries pointing to proxies that are no longer ours.
names = redis_shell.hmget('srv->name', err_cases.keys())
for name, (srv, ip) in zip(names, err_cases.iteritems()):
try:
print "Retiring", name, ip, srv
vps_util.actually_retire_proxy(name=name, ip=ip, srv=srv)
except:
alert_exception("retire non-existing proxy %s (%s), srvid %s" % (name, ip, srv))
return errors
def actually_offload_proxy(name=None, ip=None, srv=None, pipeline=None):
name, ip, srv = nameipsrv(name, ip, srv)
region = region_by_name(name)
client_table_key = region + ':clientip->srv'
packed_srv = redis_util.pack_srv(srv)
#XXX: a proxy -> {clients} index is sorely needed!
# Getting the set of clients assigned to this proxy takes a long time
# currently. Let's get it done before pulling the replacement proxy,
# so we're less likely to be left with an empty proxy if interrupted.
clients = set(pip
for pip, psrv in redis_shell.hgetall(client_table_key).iteritems()
if psrv == packed_srv)
dest = pull_from_srvq(region)
# It's still possible that we'll crash or get rebooted here, so the
# destination server will be left empty. The next closed proxy compaction
# job will find this proxy and assign some users to it or mark it for
# retirement.
dest_psrv = redis_util.pack_srv(dest.srv)
redis_shell.hmset(client_table_key, {pip: dest_psrv for pip in clients})
print "Offloaded clients from %s (%s) to %s (%s)" % (name, ip, dest.name, dest.ip)
def retire_lcs(name,
ip,
cfgcache=util.Cache(timeout=60*60,
update_fn=lambda: redis_shell.hgetall('cfgbysrv'))):
if name.startswith('fp-jp-'):
dc = 'vltok1'
elif name.startswith('fp-nl-'):
dc = 'doams3'
else:
assert False
srvs = [srv
for srv, cfg in cfgcache.get().iteritems()
if yaml.load(cfg).values()[0]['addr'].split(':')[0] == ip]
if srvs:
redis_shell.hdel('cfgbysrv', *srvs)
redis_shell.incr('srvcount')
else:
"No configs left to delete for %s." % name
redis_shell.lrem(dc + ':vpss', name)
redis_shell.incr(dc + ':vpss:version')
def actually_offload_proxy(name=None, ip=None, srv=None, pipeline=None):
name, ip, srv = nameipsrv(name, ip, srv)
region = region_by_name(name)
client_table_key = region + ':clientip->srv'
packed_srv = redis_util.pack_srv(srv)
#XXX: a proxy -> {clients} index is sorely needed!
# Getting the set of clients assigned to this proxy takes a long time
# currently. Let's get it done before pulling the replacement proxy,
# so we're less likely to be left with an empty proxy if interrupted.
clients = set(
pip for pip, psrv in redis_shell.hgetall(client_table_key).iteritems()
if psrv == packed_srv)
dest = pull_from_srvq(region)
# It's still possible that we'll crash or get rebooted here, so the
# destination server will be left empty. The next closed proxy compaction
# job will find this proxy and assign some users to it or mark it for
# retirement.
dest_psrv = redis_util.pack_srv(dest.srv)
redis_shell.hmset(client_table_key, {pip: dest_psrv for pip in clients})
print "Offloaded clients from %s (%s) to %s (%s)" % (name, ip, dest.name,
dest.ip)
def run_all_checks():
print "Fetching config data..."
srv2cfg = redis_shell.hgetall('srv->cfg')
print "Performing checks..."
cache = model.make_cache()
# This is new code, so let's test it in a cushion to start with.
try:
print "Checking that srv->cfg table is consistent with the VPS listing..."
errors = srv2cfg_consistent_with_vps_list(srv2cfg, cache)
except:
alert_exception("trying to check consistency between srv->cfg and all_vpss")
errors = []
try:
print "Check that we don't have duplicate names"
errors.extend(no_duplicate_names(cache))
except:
alert_exception("trying to check for duplicate VPS names")
print "Checking that configs start with a newline..."
errors.extend(configs_start_with_newline(srv2cfg))
regions = redis_shell.smembers('user-regions')
print "Checking that slice server entries are in srv->cfg..."
for region in regions:
print " (region %s)..." % region
errors.extend(slice_srvs_in_srv2cfg(region, srv2cfg))
print "Checking server queue size..."
for region in regions:
print " (region %s)..." % region
errors.extend(srvq_size(region))
print "Checking server queue integrity..."
for region in regions:
print " (region %s)..." % region
try:
errors.extend(srvq_integrity(region, cache=cache))
except:
alert_exception("trying to check server queue integrity")
print "Check that regional fallbacks and honeypots are in srv->cfg..."
for region in regions:
print " (region %s)..." % region
errors.extend(fallbacks_and_honeypots_in_srv_table(region, srv2cfg))
report(errors)
import yaml
from alert import alert
from redis_util import redis_shell
prefix = 'fallbacks-to-check'
try:
local_version = file(prefix + '-version').read()
except IOError:
local_version = None
remote_version = redis_shell.get('srvcount')
if local_version != remote_version:
suppress = redis_shell.smembers('checkfallbacks-suppress')
json.dump([yaml.load(cfg).values()[0]
for srv, cfg in redis_shell.hgetall('srv->cfg').iteritems()
if srv not in suppress],
file(prefix + '.json', 'w'))
file(prefix + '-version', 'w').write(remote_version)
cmd = subprocess.Popen("checkfallbacks -fallbacks %s.json -connections 20 | grep '\[failed fallback check\]'" % prefix,
shell=True,
stdout=subprocess.PIPE)
errors = list(cmd.stdout)
if errors:
for error in errors:
print error
alert(type='checkfallbacks-failures',
details={'errors': errors},
title='Proxy check failures',
text="".join(error[len('[failed fallback check] '):] + "\n"
