def get_read_table(tables):
#shortcut with 1 entry
if len(tables) == 1:
return tables[0]
#'t' is a list of engines itself. since we assume those engines
#are on the same machine, just take the first one. len(ips) may be
#< len(tables) if some tables are on the same host.
ips = dict((t[0].bind.url.host, t) for t in tables)
ip_loads = AppServiceMonitor.get_db_load(ips.keys())
total_load = 0
missing_loads = []
no_connections = []
have_loads = []
for ip in ips:
if ip not in ip_loads:
missing_loads.append(ip)
else:
load, avg_load, conns, avg_conns, max_conns = ip_loads[ip]
#prune high-connection machines
if conns < .9 * max_conns:
max_load = max(load, avg_load)
total_load += max_load
have_loads.append((ip, max_load))
else:
no_connections.append(ip)
if total_load:
avg_load = total_load / max(len(have_loads), 1)
ip_weights = [(ip, 1 - load / total_load) for ip, load in have_loads]
#if total_load is 0, which happens when have_loads is empty
else:
avg_load = 1.0
ip_weights = [(ip, 1.0 / len(have_loads)) for ip, load in have_loads]
if missing_loads or no_connections:
#add in the missing load numbers with an average weight
ip_weights.extend((ip, avg_load) for ip in missing_loads)
#add in the over-connected machines with a 1% weight
ip_weights.extend((ip, .01) for ip in no_connections)
#rebalance the weights
total_weight = sum(w[1] for w in ip_weights)
ip_weights = [(ip, weight / total_weight)
for ip, weight in ip_weights]
r = random.random()
for ip, load in ip_weights:
if r < load:
return ips[ip]
else:
r = r - load
#should never happen
print 'yer stupid'
return random.choice(tables)
def get_read_table(tables):
# short-cut for only one element
if len(tables) == 1:
return tables[0]
#'t' is a list of engines itself. since we assume those engines
#are on the same machine, just take the first one. len(ips) may be
#< len(tables) if some tables are on the same host.
ips = dict((t[0].bind.url.host, t) for t in tables)
ip_loads = AppServiceMonitor.get_db_load(ips.keys())
total_load = 0
missing_loads = []
no_connections = []
have_loads = []
for ip in ips:
if ip not in ip_loads:
missing_loads.append(ip)
else:
load, avg_load, conns, avg_conns, max_conns = ip_loads[ip]
#prune high-connection machines
#if conns < .9 * max_conns:
max_load = max(load, avg_load)
total_load += max_load
have_loads.append((ip, max_load))
#else:
# no_connections.append(ip)
if total_load:
avg_load = total_load / max(len(have_loads), 1)
ip_weights = [(ip, 1 - load / total_load) for ip, load in have_loads]
#if total_load is 0, which happens when have_loads is empty
else:
avg_load = 1.0
ip_weights = [(ip, 1.0 / len(have_loads)) for ip, load in have_loads]
if missing_loads or no_connections:
#add in the missing load numbers with an average weight
ip_weights.extend((ip, avg_load) for ip in missing_loads)
#add in the over-connected machines with a 1% weight
ip_weights.extend((ip, .01) for ip in no_connections)
#rebalance the weights
total_weight = sum(w[1] for w in ip_weights) or 1
ip_weights = [(ip, weight / total_weight) for ip, weight in ip_weights]
r = random.random()
for ip, load in ip_weights:
if r < load:
# print "db ip: %s" % str(ips[ip][0].metadata.bind.url.host)
return ips[ip]
else:
r = r - load
#should never happen
print 'yer stupid'
return random.choice(tables)
def get_read_table(self, tables):
from r2.lib.services import AppServiceMonitor
# short-cut for only one element
if len(tables) == 1:
return tables[0]
if self.dead:
tables = set(tables)
dead = set(t for t in tables if t[0].bind in self.dead)
for t in list(dead):
# TODO: tune the reconnect code. We have about 1-2
# requests per second per app, so this should
# reconnect every 50-100 seconds.
if (random.randint(1, 100) == 42
and self.test_engine(t[0].bind)):
dead.remove(t)
tables = tables - dead
#'t' is a list of engines itself. since we assume those engines
#are on the same machine, just take the first one. len(ips) may be
#< len(tables) if some tables are on the same host.
ips = dict((t[0].bind.url.host, t) for t in tables)
ip_loads = AppServiceMonitor.get_db_load(ips.keys())
total_load = 0
missing_loads = []
no_connections = []
have_loads = []
for ip in ips:
if ip not in ip_loads:
missing_loads.append(ip)
else:
load, avg_load, conns, avg_conns, max_conns = ip_loads[ip]
# remove high load machines from the pool.
if load < 100:
max_load = max(load, avg_load)
total_load += max_load
have_loads.append((ip, max_load))
else:
no_connections.append(ip)
if total_load:
avg_load = total_load / max(len(have_loads), 1)
ip_weights = [(ip, 1 - load / total_load)
for ip, load in have_loads]
#if total_load is 0, which happens when have_loads is empty
else:
avg_load = 1.0
ip_weights = [(ip, 1.0 / len(have_loads))
for ip, load in have_loads]
if missing_loads or no_connections:
#add in the missing load numbers with an average weight
ip_weights.extend((ip, avg_load) for ip in missing_loads)
#add in the over-connected machines with a 1% weight
ip_weights.extend((ip, .01) for ip in no_connections)
#rebalance the weights
total_weight = sum(w[1] for w in ip_weights) or 1
ip_weights = [(ip, weight / total_weight) for ip, weight in ip_weights]
r = random.random()
for ip, load in ip_weights:
if r < load:
# print "db ip: %s" % str(ips[ip][0].metadata.bind.url.host)
return ips[ip]
r = r - load
#should never happen
print 'yer stupid'
return random.choice(list(tables))
def get_read_table(self, tables):
from r2.lib.services import AppServiceMonitor
# short-cut for only one element
if len(tables) == 1:
return tables[0]
if self.dead:
tables = set(tables)
dead = set(t for t in tables if t[0].bind in self.dead)
for t in list(dead):
# TODO: tune the reconnect code. We have about 1-2
# requests per second per app, so this should
# reconnect every 50-100 seconds.
if (random.randint(1,100) == 42 and
self.test_engine(t[0].bind)):
dead.remove(t)
tables = tables - dead
#'t' is a list of engines itself. since we assume those engines
#are on the same machine, just take the first one. len(ips) may be
#< len(tables) if some tables are on the same host.
ips = dict((t[0].bind.url.host, t) for t in tables)
ip_loads = AppServiceMonitor.get_db_load(ips.keys())
total_load = 0
missing_loads = []
no_connections = []
have_loads = []
for ip in ips:
if ip not in ip_loads:
missing_loads.append(ip)
else:
load, avg_load, conns, avg_conns, max_conns = ip_loads[ip]
# remove high load machines from the pool.
if load < 100:
max_load = max(load, avg_load)
total_load += max_load
have_loads.append((ip, max_load))
else:
no_connections.append(ip)
if total_load:
avg_load = total_load / max(len(have_loads), 1)
ip_weights = [(ip, 1 - load / total_load) for ip, load in have_loads]
#if total_load is 0, which happens when have_loads is empty
else:
avg_load = 1.0
ip_weights = [(ip, 1.0 / len(have_loads)) for ip, load in have_loads]
if missing_loads or no_connections:
#add in the missing load numbers with an average weight
ip_weights.extend((ip, avg_load) for ip in missing_loads)
#add in the over-connected machines with a 1% weight
ip_weights.extend((ip, .01) for ip in no_connections)
#rebalance the weights
total_weight = sum(w[1] for w in ip_weights) or 1
ip_weights = [(ip, weight / total_weight)
for ip, weight in ip_weights]
r = random.random()
for ip, load in ip_weights:
if r < load:
# print "db ip: %s" % str(ips[ip][0].metadata.bind.url.host)
return ips[ip]
r = r - load
#should never happen
print 'yer stupid'
return random.choice(list(tables))
def get_read_table(self, tables):
from r2.lib.services import AppServiceMonitor
# short-cut for only one element
if len(tables) == 1:
return tables[0]
if self.dead:
tables = set(tables)
dead = set(t for t in tables if t[0].bind in self.dead)
for t in list(dead):
# TODO: tune the reconnect code. We have about 1-2
# requests per second per app, so this should
# reconnect every 50-100 seconds.
#
# random.random() generates a random float <= 1.
# db_dead_reconnect_prob is defined in the ini
# 0.01 makes a 1/100 chance of attempting a reconnect
# 1.00 makes a 1/1 chance.
rand = random.random()
logger.debug("if {0} < {1} , we are trying to reconnect...".format(rand, self.db_dead_reconnect_prob))
if rand < self.db_dead_reconnect_prob:
if self.test_engine(t[0].bind):
dead.remove(t)
#only apply changes to tables if there are changes to apply
if dead:
tables = tables - dead
#'t' is a list of engines itself. since we assume those engines
#are on the same machine, just take the first one. len(ips) may be
#< len(tables) if some tables are on the same host.
ips = dict((t[0].bind.url.host, t) for t in tables)
ip_loads = AppServiceMonitor.get_db_load(ips.keys())
total_load = 0
missing_loads = []
no_connections = []
have_loads = []
for ip in ips:
if ip not in ip_loads:
missing_loads.append(ip)
else:
load, avg_load, conns, avg_conns, max_conns = ip_loads[ip]
# remove high load machines from the pool.
if load < 100:
max_load = max(load, avg_load)
total_load += max_load
have_loads.append((ip, max_load))
else:
no_connections.append(ip)
if total_load:
avg_load = total_load / max(len(have_loads), 1)
ip_weights = [(ip, 1 - load / total_load) for ip, load in have_loads]
#if total_load is 0, which happens when have_loads is empty
else:
avg_load = 1.0
ip_weights = [(ip, 1.0 / len(have_loads)) for ip, load in have_loads]
if missing_loads or no_connections:
#add in the missing load numbers with an average weight
ip_weights.extend((ip, avg_load) for ip in missing_loads)
#add in the over-connected machines with a 1% weight
ip_weights.extend((ip, .01) for ip in no_connections)
#rebalance the weights
total_weight = sum(w[1] for w in ip_weights) or 1
ip_weights = [(ip, weight / total_weight)
for ip, weight in ip_weights]
r = random.random()
for ip, load in ip_weights:
if r < load:
# print "db ip: %s" % str(ips[ip][0].metadata.bind.url.host)
return ips[ip]
r = r - load
#should never happen
logger.error("""I couldn't find any usable PGSQLs anymore. Maybe it is down or maybe I just think it is down.
Restarting reddit or postgresql may be a good short-term fix for this. Please examine the logs more thorougly
to attempt to find the time I lose all record of usable connections and fix whatever causes this.""")
return random.choice(list(tables))
def Run(srvname, *a, **kw):
args = {}
if kw.has_key('queue_length_max'):
args['queue_length_max'] = kw.pop('queue_length_max')
AppServiceMonitor(**args).monitor(srvname, *a, **kw)
def Alert(restart_list=['MEM', 'CPU'],
alert_recipients=[nerds_email],
alert_sender=nerds_email,
cpu_limit=99,
mem_limit=8,
smtpserver=smtp_server,
test=False):
p = re.compile("newreddit(\d+)")
cache_key = 'already_alerted_'
for host in AppServiceMonitor(g.monitored_servers):
for service in host:
# cpu values
cpu = [service.cpu(x) for x in (0, 5, 60, 300)]
output = "\nCPU: " + ' '.join("%6.2f%%" % x for x in cpu)
output += "\nMEMORY: %6.2f%%" % service.mem()
service_name = "%s %s" % (host.host, service.name)
# is this service pegged?
mem_pegged = ('MEM' in restart_list and service.mem() > mem_limit)
need_restart = (('CPU' in restart_list and all(x >= cpu_limit
for x in cpu))
or mem_pegged)
if (need_restart):
mesg = ("To: " + ', '.join(alert_recipients) + "\nSubject: " +
service_name + " needs attention\n\n" + service_name +
(" is out of mem: " if mem_pegged else " is pegged:") +
output)
m = p.match(service.name)
# If we can restart this process, we do it here
if m:
proc_number = str(m.groups()[0])
cmd = "/usr/local/bin/push -h " + \
host.host + " -r " + proc_number
if test:
print(
"would have restarted the app with command '%s'" %
cmd)
else:
result = os.popen3(cmd)[2].read()
# We override the other message to show we restarted it
mesg = ("To: [email protected]\n" + "Subject: " +
"Process " + proc_number + " on " + host.host +
" was automatically restarted " +
"due to the following:\n\n" + output + "\n\n" +
"Here was the output:\n" + result)
# Uncomment this to disable restart messages
#mesg = ""
last_alerted = g.rendercache.get(cache_key + service_name) or 0
#last_alerted = 0
if mesg is not "":
if test:
print "would have sent email\n '%s'" % mesg
elif (time.time() - last_alerted > 300):
g.rendercache.set(cache_key + service_name,
time.time())
session = smtplib.SMTP(smtpserver)
smtpresult = session.sendmail(alert_sender,
alert_recipients, mesg)
session.quit()
