def run():
cm = vps_util.my_cm()
region = vps_util.my_region()
print "Starting retire server at cloudmaster %s, region %s." % (cm, region)
qname = cm + ":retireq"
destroy_qname = cm + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
is_baked_in = redis_shell.sismember(region + ":bakedin-names",
name)
txn = redis_shell.pipeline()
if is_baked_in:
print "Not retiring baked-in server %s (%s)" % (name, ip)
else:
print "Retiring", name, ip
vps_util.actually_retire_proxy(name, ip, txn)
remover(txn)
if not is_baked_in:
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
else:
time.sleep(10)
def run():
dc = os.getenv("DC")
print "Using datacenter", dc, ", MAXPROCS", repr(MAXPROCS)
qname = dc + ":srvreqq"
reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT)
procq = multiprocessing.Queue()
pending = {}
def kill_task(reqid):
print "Killing timed out process and vps..."
task = pending.pop(reqid)
task['proc'].terminate()
proc = multiprocessing.Process(target=vps_shell(dc).destroy_vps,
args=(task['name'], ))
proc.daemon = True
proc.start()
while True:
while not procq.empty():
try:
result = procq.get(False)
print "Got result:", result
task = pending.get(result['reqid'])
if task and task['name'] == result['name']:
del pending[result['reqid']]
upload_cfg(redis_shell, dc, result['access_data'])
register_vps(redis_shell, dc, task['name'])
task['remove_req']()
except Empty:
print "Wat?"
break
if len(pending) < MAXPROCS:
reqid, remover = reqq.next_job()
if reqid:
print "Got request", reqid
if reqid in pending:
print "Killing task %s because of queue timeout" % reqid
kill_task(reqid)
name = get_lcs_name(dc, redis_shell)
proc = multiprocessing.Process(target=launch_one_server,
args=(procq, reqid, dc, name))
proc.daemon = True
pending[reqid] = {
'name': name,
'proc': proc,
'starttime': time.time(),
'remove_req': remover
}
print "Starting process to launch", name
proc.start()
else:
# Since we're not checking the queue when we've maxed out our
# processes, we need to manually check for expired tasks.
for reqid, d in pending.items():
if time.time() - d['starttime'] > LAUNCH_TIMEOUT:
print "Killing task %s because of local timeout" % reqid
kill_task(reqid)
time.sleep(10)
def run():
dc = os.getenv("DC")
print "Using datacenter", dc
qname = dc + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
name, remover = q.next_job()
if name:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
time.sleep(10)
def run():
qname = vps_util.my_cm() + ":destroyq"
region = vps_util.my_region()
q = redisq.Queue(qname, redis_shell, TIMEOUT)
print "Starting destroy service in cloudmaster %s, region %s." % (
vps_util.my_cm(), region)
while True:
name, remover = q.next_job()
if name:
if redis_shell.sismember(region + ":bakedin-names", name):
print "Not retiring baked-in server", name
else:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
else:
time.sleep(10)
def run():
region = vps_util.my_region()
print "Starting offload server for region %s." % region
qname = region + ":offloadq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Offloading users from %s (%s)" % (name, ip)
txn = redis_shell.pipeline()
vps_util.actually_offload_proxy(name, ip, pipeline=txn)
remover(txn)
cm = vps_util.cm_by_name(name)
txn.lpush(cm + ':retireq', task)
txn.execute()
else:
time.sleep(10)
def run():
dc = os.getenv("DC")
print "Using datacenter", dc
qname = dc + ":retireq"
destroy_qname = dc + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Retiring", name, ip
vps_util.retire_lcs(name, ip)
txn = redis_shell.pipeline()
remover(txn)
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
time.sleep(10)
def run():
region = vps_util.my_region()
print "Starting offload server for region %s." % region
qname = region + ":offloadq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
print "Offloading users from %s (%s)" % (name, ip)
txn = redis_shell.pipeline()
try:
vps_util.actually_offload_proxy(name, ip, pipeline=txn)
except vps_util.ProxyGone:
print >> sys.stderr, "Tried to offload no longer existing proxy %s (%s)" % (
name, ip)
remover(redis_shell)
continue
remover(txn)
cm = vps_util.cm_by_name(name)
txn.lpush(cm + ':retireq', task)
txn.execute()
else:
time.sleep(10)
def run():
qname = QPREFIX + ":srvreqq"
print "Serving queue", qname, ", MAXPROCS:", repr(MAXPROCS)
quarantine = CM + ":quarantined_vpss"
reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT)
procq = multiprocessing.Queue()
pending = {}
def kill_task(reqid):
print "Killing timed out process and vps..."
task = pending.pop(reqid)
task['proc'].terminate()
proc = multiprocessing.Process(target=vps_shell.destroy_vps,
args=(task['name'],))
proc.daemon = True
proc.start()
while True:
# If the request queue is totally empty (no tasks enqueued or even in
# progress), flush the quarantine queue into the destroy queue.
if redis_shell.llen(qname) == 1: # 1 for the redisq sentinel entry
names = redis_shell.smembers(quarantine)
if names:
print "Flushing %s VPSs from quarantine." % len(names)
p = redis_shell.pipeline()
p.srem(quarantine, *names)
p.lpush(CM + ":destroyq", *names)
p.execute()
while not procq.empty():
try:
result = procq.get(False)
print "Got result:", result
task = pending.get(result['reqid'])
if task and task['name'] == result['name']:
p = redis_shell.pipeline()
if result['blocked']:
print "Quarantining %(name)s (%(ip)s)." % result
p.sadd(quarantine, result['name'])
p.incr(CM + ":blocked_vps_count") # stats
# We'll remove the original request anyway because we
# don't want it to stay around until timeout. Insert a
# new one to replace it instead.
reqid = redis_shell.incr('srvcount')
p.lpush(qname, reqid)
else:
p.incr(CM + ":unblocked_vps_count") # stats
del pending[result['reqid']]
vps_util.enqueue_cfg(result['name'], result['access_data'], result['srvq'])
register_vps(task['name'])
task['remove_req'](p)
p.execute()
except Empty:
print "Wat?"
break
if len(pending) < MAXPROCS:
req_string, remover = reqq.next_job()
if req_string:
print "Got request", req_string
req = json.loads(req_string)
if isinstance(req, int):
# Transition: support the old format while we are updating
# the config server etc.
req = {'id': req, 'srvq': QPREFIX + ':srvq'}
req_string = json.dumps(req)
reqid = req['id']
if reqid in pending:
print "Killing task %s because of queue timeout" % reqid
kill_task(reqid)
name = new_proxy_name(req)
proc = multiprocessing.Process(target=launch_one_server,
args=(procq,
reqid,
name,
req_string))
proc.daemon = True
pending[reqid] = {
'name': name,
'proc': proc,
'starttime': time.time(),
'remove_req': remover}
print "Starting process to launch", name
proc.start()
else:
# Since we're not checking the queue when we've maxed out our
# processes, we need to manually check for expired tasks.
for reqid, d in pending.items():
if time.time() - d['starttime'] > LAUNCH_TIMEOUT:
print "Killing task %s because of local timeout" % reqid
kill_task(reqid)
time.sleep(10)
