def run(): qname = vps_util.my_cm() + ":destroyq" region = vps_util.my_region() q = redisq.Queue(qname, redis_shell, TIMEOUT) print "Starting destroy service in cloudmaster %s, region %s." % (vps_util.my_cm(), region) while True: name, remover = q.next_job() if name: if redis_shell.sismember(region + ":bakedin-names", name): print "Not retiring baked-in server", name else: print "Destroying", name vps_util.destroy_vps(name) remover() else: time.sleep(10)
def run(): cm = vps_util.my_cm() region = vps_util.my_region() print "Starting retire server at cloudmaster %s, region %s." % (cm, region) qname = cm + ":retireq" destroy_qname = cm + ":destroyq" q = redisq.Queue(qname, redis_shell, TIMEOUT) while True: task, remover = q.next_job() if task: name, ip = task.split('|') is_baked_in = redis_shell.sismember(region + ":bakedin-names", name) txn = redis_shell.pipeline() if is_baked_in: print "Not retiring baked-in server %s (%s)" % (name, ip) else: print "Retiring", name, ip vps_util.actually_retire_proxy(name, ip, txn) remover(txn) if not is_baked_in: # Introduce the job with the timestamp already filled in, so it will # only be pulled when it 'expires'. This effectively adds a delay to # give clients some time to move over to their new server before we # actually destroy the old one. txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time()))) txn.execute() else: time.sleep(10)
def create_vps(name): vps_util.save_pillar(name) plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB" out = subprocess.check_output(["salt-cloud", "-p", plan, name]) # Uberhack: XXX update with salt version... d = yaml.load(out[out.rfind(name + ":"):].replace("----------", "").replace("|_", "-")).values()[0] return d['name'], d['networks']['v4'][1]['ip_address']
def run(): qname = vps_util.my_cm() + ":destroyq" region = vps_util.my_region() q = redisq.Queue(qname, redis_shell, TIMEOUT) print "Starting destroy service in cloudmaster %s, region %s." % ( vps_util.my_cm(), region) while True: name, remover = q.next_job() if name: if redis_shell.sismember(region + ":bakedin-names", name): print "Not retiring baked-in server", name else: print "Destroying", name vps_util.destroy_vps(name) remover() else: time.sleep(10)
def create_vps(name): vps_util.save_pillar(name) plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB" out = subprocess.check_output(["salt-cloud", "-p", plan, name]) # Uberhack: XXX update with salt version... d = yaml.load(out[out.rfind(name + ":"):].replace( "----------", "").replace("|_", "-")).values()[0] return d['name'], d['networks']['v4'][1]['ip_address']
def retire_server(msg): if os.path.exists(retire_flag_filename): print "Not retiring myself again." return redis_shell.lpush(vps_util.my_cm() + ':retireq', '%s|%s' % (instance_id, ip)) flag_as_done(retire_flag_filename) send_alarm("Chained proxy RETIRED", " retired because I " + msg)
def create_vps(name, req={}, plan=None): vps_util.save_pillar(name, req) if plan is None: plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB" out = subprocess.check_output(["salt-cloud", "-p", plan, name]) # Uberhack: XXX update with salt version... d = yaml.load(out[out.rfind(name + ":"):].replace( "----------", "").replace("|_", "-")).values()[0] return {'name': d['name'], 'ip': d['public_ips'][0]}
def create_vps(name, req={}, plan=None): vps_util.save_pillar(name, req) if plan is None: plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB" out = subprocess.check_output(["salt-cloud", "-p", plan, name]) # Uberhack: XXX update with salt version... d = yaml.load(out[out.rfind(name + ":"):].replace("----------", "").replace("|_", "-")).values()[0] return {'name': d['name'], 'ip': d['public_ips'][0]}
def create_vps(label, req): vps_util.save_pillar(label, req) dc = vps_util.dc_by_cm(vps_util.my_cm()) subid = vultr.server_create(vultr_dcid[dc], default_plan[dc], ubuntu14_04_64bit, label=label, sshkeyid=cloudmaster_keyid, enable_ipv6='yes', enable_private_network="yes")['SUBID'] for _ in xrange(30): time.sleep(10) d = try_vultr_cmd(vultr.server_list, subid) ip = d.get('main_ip') if ip and util.ipre.match(ip): d['ip'] = ip return d raise RuntimeError("Couldn't get the new VPS's IP")
def launch_config_server(): # XXX: this doesn't perform a complete setup yet. name = vps_util.new_vps_name('cs') d = do_util.create_vps(name, plan=vps_util.dc_by_cm(vps_util.my_cm()) + '_2GB') if not vps_util.highstate_pid(name): print("Highstate not running yet; waiting for a bit just in case...") time.sleep(10) while vps_util.highstate_pid(name): print("Highstate still running...") time.sleep(10) ip = d['ip'] print "Checking for completion..." while True: try: resp = requests.get(('http://%s/proxies.yaml.gz' % ip), timeout=5) if resp.ok: break except requests.exceptions.RequestException: pass print "Server setup not complete yet; retrying..." time.sleep(5) print "%s (%s) is up." % (name, ip)
import multiprocessing from Queue import Empty import os import time import traceback import yaml import json from alert import send_to_slack from redis_util import redis_shell import redisq import vps_util CM = vps_util.my_cm() REGION = vps_util.my_region() scope = os.environ['QSCOPE'] if scope == 'REGION': QPREFIX = REGION elif scope == 'CM': QPREFIX = CM else: assert False MAXPROCS = int(os.getenv('MAXPROCS')) LAUNCH_TIMEOUT = 30 * 60 vps_shell = vps_util.vps_shell(CM) def run():