def run():
cm = vps_util.my_cm()
region = vps_util.my_region()
print "Starting retire server at cloudmaster %s, region %s." % (cm, region)
qname = cm + ":retireq"
destroy_qname = cm + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
is_baked_in = redis_shell.sismember(region + ":bakedin-names", name)
txn = redis_shell.pipeline()
if is_baked_in:
print "Not retiring baked-in server %s (%s)" % (name, ip)
else:
print "Retiring", name, ip
vps_util.actually_retire_proxy(name, ip, txn)
remover(txn)
if not is_baked_in:
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
else:
time.sleep(10)
def run():
cm = vps_util.my_cm()
region = vps_util.my_region()
print "Starting retire server at cloudmaster %s, region %s." % (cm, region)
qname = cm + ":retireq"
destroy_qname = cm + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
is_baked_in = redis_shell.sismember(region + ":bakedin-names",
name)
txn = redis_shell.pipeline()
if is_baked_in:
print "Not retiring baked-in server %s (%s)" % (name, ip)
else:
print "Retiring", name, ip
vps_util.actually_retire_proxy(name, ip, txn)
remover(txn)
if not is_baked_in:
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
else:
time.sleep(10)
def run():
qname = vps_util.my_cm() + ":destroyq"
region = vps_util.my_region()
q = redisq.Queue(qname, redis_shell, TIMEOUT)
print "Starting destroy service in cloudmaster %s, region %s." % (vps_util.my_cm(), region)
while True:
name, remover = q.next_job()
if name:
if redis_shell.sismember(region + ":bakedin-names", name):
print "Not retiring baked-in server", name
else:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
else:
time.sleep(10)
def run():
qname = vps_util.my_cm() + ":destroyq"
region = vps_util.my_region()
q = redisq.Queue(qname, redis_shell, TIMEOUT)
print "Starting destroy service in cloudmaster %s, region %s." % (
vps_util.my_cm(), region)
while True:
name, remover = q.next_job()
if name:
if redis_shell.sismember(region + ":bakedin-names", name):
print "Not retiring baked-in server", name
else:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
else:
time.sleep(10)
def run():
region = vps_util.my_region()
print "Starting offload server for region %s." % region
qname = region + ":offloadq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Offloading users from %s (%s)" % (name, ip)
txn = redis_shell.pipeline()
vps_util.actually_offload_proxy(name, ip, pipeline=txn)
remover(txn)
cm = vps_util.cm_by_name(name)
txn.lpush(cm + ':retireq', task)
txn.execute()
else:
time.sleep(10)
def run():
region = vps_util.my_region()
print "Starting offload server for region %s." % region
qname = region + ":offloadq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Offloading users from %s (%s)" % (name, ip)
txn = redis_shell.pipeline()
vps_util.actually_offload_proxy(name, ip, pipeline=txn)
remover(txn)
cm = vps_util.cm_by_name(name)
txn.lpush(cm + ':retireq', task)
txn.execute()
else:
time.sleep(10)
def run():
region = vps_util.my_region()
print "Starting offload server for region %s." % region
qname = region + ":offloadq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Offloading users from %s (%s)" % (name, ip)
txn = redis_shell.pipeline()
try:
vps_util.actually_offload_proxy(name, ip, pipeline=txn)
except vps_util.ProxyGone:
print >> sys.stderr, "Tried to offload no longer existing proxy %s (%s)" % (
name, ip)
remover(redis_shell)
continue
remover(txn)
cm = vps_util.cm_by_name(name)
txn.lpush(cm + ':retireq', task)
txn.execute()
else:
time.sleep(10)
from Queue import Empty
import os
import time
import traceback
import yaml
import json
from alert import send_to_slack
from redis_util import redis_shell
import redisq
import vps_util
CM = vps_util.my_cm()
REGION = vps_util.my_region()
scope = os.environ['QSCOPE']
if scope == 'REGION':
QPREFIX = REGION
elif scope == 'CM':
QPREFIX = CM
else:
assert False
MAXPROCS = int(os.getenv('MAXPROCS'))
LAUNCH_TIMEOUT = 30 * 60
vps_shell = vps_util.vps_shell(CM)
def run():
qname = QPREFIX + ":srvreqq"
