def run(): cm = vps_util.my_cm() region = vps_util.my_region() print "Starting retire server at cloudmaster %s, region %s." % (cm, region) qname = cm + ":retireq" destroy_qname = cm + ":destroyq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: task, remover = q.next_job() if task: name, ip = task.split('|') is_baked_in = redis_shell.sismember(region + ":bakedin-names", name) txn = redis_shell.pipeline() if is_baked_in: print "Not retiring baked-in server %s (%s)" % (name, ip) else: print "Retiring", name, ip vps_util.actually_retire_proxy(name, ip, txn) remover(txn) if not is_baked_in: # Introduce the job with the timestamp already filled in, so it will # only be pulled when it 'expires'. This effectively adds a delay to # give clients some time to move over to their new server before we # actually destroy the old one. txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time()))) txn.execute() else: time.sleep(10)
def run(): dc = os.getenv("DC") print "Using datacenter", dc, ", MAXPROCS", repr(MAXPROCS) qname = dc + ":srvreqq" reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT) procq = multiprocessing.Queue() pending = {} def kill_task(reqid): print "Killing timed out process and vps..." task = pending.pop(reqid) task['proc'].terminate() proc = multiprocessing.Process(target=vps_shell(dc).destroy_vps, args=(task['name'], )) proc.daemon = True proc.start() while True: while not procq.empty(): try: result = procq.get(False) print "Got result:", result task = pending.get(result['reqid']) if task and task['name'] == result['name']: del pending[result['reqid']] upload_cfg(redis_shell, dc, result['access_data']) register_vps(redis_shell, dc, task['name']) task['remove_req']() except Empty: print "Wat?" break if len(pending) < MAXPROCS: reqid, remover = reqq.next_job() if reqid: print "Got request", reqid if reqid in pending: print "Killing task %s because of queue timeout" % reqid kill_task(reqid) name = get_lcs_name(dc, redis_shell) proc = multiprocessing.Process(target=launch_one_server, args=(procq, reqid, dc, name)) proc.daemon = True pending[reqid] = { 'name': name, 'proc': proc, 'starttime': time.time(), 'remove_req': remover } print "Starting process to launch", name proc.start() else: # Since we're not checking the queue when we've maxed out our # processes, we need to manually check for expired tasks. for reqid, d in pending.items(): if time.time() - d['starttime'] > LAUNCH_TIMEOUT: print "Killing task %s because of local timeout" % reqid kill_task(reqid) time.sleep(10)
def run(): dc = os.getenv("DC") print "Using datacenter", dc qname = dc + ":destroyq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: name, remover = q.next_job() if name: print "Destroying", name vps_util.destroy_vps(name) remover() time.sleep(10)
def run(): qname = vps_util.my_cm() + ":destroyq" region = vps_util.my_region() q = redisq.Queue(qname, redis_shell, TIMEOUT) print "Starting destroy service in cloudmaster %s, region %s." % ( vps_util.my_cm(), region) while True: name, remover = q.next_job() if name: if redis_shell.sismember(region + ":bakedin-names", name): print "Not retiring baked-in server", name else: print "Destroying", name vps_util.destroy_vps(name) remover() else: time.sleep(10)
def run(): region = vps_util.my_region() print "Starting offload server for region %s." % region qname = region + ":offloadq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: task, remover = q.next_job() if task: name, ip = task.split('|') print "Offloading users from %s (%s)" % (name, ip) txn = redis_shell.pipeline() vps_util.actually_offload_proxy(name, ip, pipeline=txn) remover(txn) cm = vps_util.cm_by_name(name) txn.lpush(cm + ':retireq', task) txn.execute() else: time.sleep(10)
def run(): dc = os.getenv("DC") print "Using datacenter", dc qname = dc + ":retireq" destroy_qname = dc + ":destroyq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: task, remover = q.next_job() if task: name, ip = task.split('|') print "Retiring", name, ip vps_util.retire_lcs(name, ip) txn = redis_shell.pipeline() remover(txn) # Introduce the job with the timestamp already filled in, so it will # only be pulled when it 'expires'. This effectively adds a delay to # give clients some time to move over to their new server before we # actually destroy the old one. txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time()))) txn.execute() time.sleep(10)
def run(): region = vps_util.my_region() print "Starting offload server for region %s." % region qname = region + ":offloadq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: task, remover = q.next_job() if task: name, ip = task.split('|') print "Offloading users from %s (%s)" % (name, ip) txn = redis_shell.pipeline() try: vps_util.actually_offload_proxy(name, ip, pipeline=txn) except vps_util.ProxyGone: print >> sys.stderr, "Tried to offload no longer existing proxy %s (%s)" % ( name, ip) remover(redis_shell) continue remover(txn) cm = vps_util.cm_by_name(name) txn.lpush(cm + ':retireq', task) txn.execute() else: time.sleep(10)
def run(): qname = QPREFIX + ":srvreqq" print "Serving queue", qname, ", MAXPROCS:", repr(MAXPROCS) quarantine = CM + ":quarantined_vpss" reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT) procq = multiprocessing.Queue() pending = {} def kill_task(reqid): print "Killing timed out process and vps..." task = pending.pop(reqid) task['proc'].terminate() proc = multiprocessing.Process(target=vps_shell.destroy_vps, args=(task['name'],)) proc.daemon = True proc.start() while True: # If the request queue is totally empty (no tasks enqueued or even in # progress), flush the quarantine queue into the destroy queue. if redis_shell.llen(qname) == 1: # 1 for the redisq sentinel entry names = redis_shell.smembers(quarantine) if names: print "Flushing %s VPSs from quarantine." % len(names) p = redis_shell.pipeline() p.srem(quarantine, *names) p.lpush(CM + ":destroyq", *names) p.execute() while not procq.empty(): try: result = procq.get(False) print "Got result:", result task = pending.get(result['reqid']) if task and task['name'] == result['name']: p = redis_shell.pipeline() if result['blocked']: print "Quarantining %(name)s (%(ip)s)." % result p.sadd(quarantine, result['name']) p.incr(CM + ":blocked_vps_count") # stats # We'll remove the original request anyway because we # don't want it to stay around until timeout. Insert a # new one to replace it instead. reqid = redis_shell.incr('srvcount') p.lpush(qname, reqid) else: p.incr(CM + ":unblocked_vps_count") # stats del pending[result['reqid']] vps_util.enqueue_cfg(result['name'], result['access_data'], result['srvq']) register_vps(task['name']) task['remove_req'](p) p.execute() except Empty: print "Wat?" break if len(pending) < MAXPROCS: req_string, remover = reqq.next_job() if req_string: print "Got request", req_string req = json.loads(req_string) if isinstance(req, int): # Transition: support the old format while we are updating # the config server etc. req = {'id': req, 'srvq': QPREFIX + ':srvq'} req_string = json.dumps(req) reqid = req['id'] if reqid in pending: print "Killing task %s because of queue timeout" % reqid kill_task(reqid) name = new_proxy_name(req) proc = multiprocessing.Process(target=launch_one_server, args=(procq, reqid, name, req_string)) proc.daemon = True pending[reqid] = { 'name': name, 'proc': proc, 'starttime': time.time(), 'remove_req': remover} print "Starting process to launch", name proc.start() else: # Since we're not checking the queue when we've maxed out our # processes, we need to manually check for expired tasks. for reqid, d in pending.items(): if time.time() - d['starttime'] > LAUNCH_TIMEOUT: print "Killing task %s because of local timeout" % reqid kill_task(reqid) time.sleep(10)