def run():
qname = vps_util.my_cm() + ":destroyq"
region = vps_util.my_region()
q = redisq.Queue(qname, redis_shell, TIMEOUT)
print "Starting destroy service in cloudmaster %s, region %s." % (vps_util.my_cm(), region)
while True:
name, remover = q.next_job()
if name:
if redis_shell.sismember(region + ":bakedin-names", name):
print "Not retiring baked-in server", name
else:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
else:
time.sleep(10)
def run():
cm = vps_util.my_cm()
region = vps_util.my_region()
print "Starting retire server at cloudmaster %s, region %s." % (cm, region)
qname = cm + ":retireq"
destroy_qname = cm + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
is_baked_in = redis_shell.sismember(region + ":bakedin-names", name)
txn = redis_shell.pipeline()
if is_baked_in:
print "Not retiring baked-in server %s (%s)" % (name, ip)
else:
print "Retiring", name, ip
vps_util.actually_retire_proxy(name, ip, txn)
remover(txn)
if not is_baked_in:
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
else:
time.sleep(10)
def create_vps(name):
vps_util.save_pillar(name)
plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB"
out = subprocess.check_output(["salt-cloud", "-p", plan, name])
# Uberhack: XXX update with salt version...
d = yaml.load(out[out.rfind(name + ":"):].replace("----------", "").replace("|_", "-")).values()[0]
return d['name'], d['networks']['v4'][1]['ip_address']
def run():
cm = vps_util.my_cm()
region = vps_util.my_region()
print "Starting retire server at cloudmaster %s, region %s." % (cm, region)
qname = cm + ":retireq"
destroy_qname = cm + ":destroyq"
q = redisq.Queue(qname, redis_shell, TIMEOUT)
while True:
task, remover = q.next_job()
if task:
name, ip = task.split('|')
is_baked_in = redis_shell.sismember(region + ":bakedin-names",
name)
txn = redis_shell.pipeline()
if is_baked_in:
print "Not retiring baked-in server %s (%s)" % (name, ip)
else:
print "Retiring", name, ip
vps_util.actually_retire_proxy(name, ip, txn)
remover(txn)
if not is_baked_in:
# Introduce the job with the timestamp already filled in, so it will
# only be pulled when it 'expires'. This effectively adds a delay to
# give clients some time to move over to their new server before we
# actually destroy the old one.
txn.lpush(destroy_qname, "%s*%s" % (name, int(time.time())))
txn.execute()
else:
time.sleep(10)
def run():
qname = vps_util.my_cm() + ":destroyq"
region = vps_util.my_region()
q = redisq.Queue(qname, redis_shell, TIMEOUT)
print "Starting destroy service in cloudmaster %s, region %s." % (
vps_util.my_cm(), region)
while True:
name, remover = q.next_job()
if name:
if redis_shell.sismember(region + ":bakedin-names", name):
print "Not retiring baked-in server", name
else:
print "Destroying", name
vps_util.destroy_vps(name)
remover()
else:
time.sleep(10)
def create_vps(name):
vps_util.save_pillar(name)
plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB"
out = subprocess.check_output(["salt-cloud", "-p", plan, name])
# Uberhack: XXX update with salt version...
d = yaml.load(out[out.rfind(name + ":"):].replace(
"----------", "").replace("|_", "-")).values()[0]
return d['name'], d['networks']['v4'][1]['ip_address']
def retire_server(msg):
if os.path.exists(retire_flag_filename):
print "Not retiring myself again."
return
redis_shell.lpush(vps_util.my_cm() + ':retireq', '%s|%s' % (instance_id, ip))
flag_as_done(retire_flag_filename)
send_alarm("Chained proxy RETIRED",
" retired because I " + msg)
def create_vps(name, req={}, plan=None):
vps_util.save_pillar(name, req)
if plan is None:
plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB"
out = subprocess.check_output(["salt-cloud", "-p", plan, name])
# Uberhack: XXX update with salt version...
d = yaml.load(out[out.rfind(name + ":"):].replace(
"----------", "").replace("|_", "-")).values()[0]
return {'name': d['name'], 'ip': d['public_ips'][0]}
def create_vps(name, req={}, plan=None):
vps_util.save_pillar(name, req)
if plan is None:
plan = vps_util.dc_by_cm(vps_util.my_cm()) + "_1GB"
out = subprocess.check_output(["salt-cloud", "-p", plan, name])
# Uberhack: XXX update with salt version...
d = yaml.load(out[out.rfind(name + ":"):].replace("----------", "").replace("|_", "-")).values()[0]
return {'name': d['name'],
'ip': d['public_ips'][0]}
def create_vps(label, req):
vps_util.save_pillar(label, req)
dc = vps_util.dc_by_cm(vps_util.my_cm())
subid = vultr.server_create(vultr_dcid[dc],
default_plan[dc],
ubuntu14_04_64bit,
label=label,
sshkeyid=cloudmaster_keyid,
enable_ipv6='yes',
enable_private_network="yes")['SUBID']
for _ in xrange(30):
time.sleep(10)
d = try_vultr_cmd(vultr.server_list, subid)
ip = d.get('main_ip')
if ip and util.ipre.match(ip):
d['ip'] = ip
return d
raise RuntimeError("Couldn't get the new VPS's IP")
def create_vps(label, req):
vps_util.save_pillar(label, req)
dc = vps_util.dc_by_cm(vps_util.my_cm())
subid = vultr.server_create(vultr_dcid[dc],
default_plan[dc],
ubuntu14_04_64bit,
label=label,
sshkeyid=cloudmaster_keyid,
enable_ipv6='yes',
enable_private_network="yes")['SUBID']
for _ in xrange(30):
time.sleep(10)
d = try_vultr_cmd(vultr.server_list, subid)
ip = d.get('main_ip')
if ip and util.ipre.match(ip):
d['ip'] = ip
return d
raise RuntimeError("Couldn't get the new VPS's IP")
def launch_config_server():
# XXX: this doesn't perform a complete setup yet.
name = vps_util.new_vps_name('cs')
d = do_util.create_vps(name, plan=vps_util.dc_by_cm(vps_util.my_cm()) + '_2GB')
if not vps_util.highstate_pid(name):
print("Highstate not running yet; waiting for a bit just in case...")
time.sleep(10)
while vps_util.highstate_pid(name):
print("Highstate still running...")
time.sleep(10)
ip = d['ip']
print "Checking for completion..."
while True:
try:
resp = requests.get(('http://%s/proxies.yaml.gz' % ip),
timeout=5)
if resp.ok:
break
except requests.exceptions.RequestException:
pass
print "Server setup not complete yet; retrying..."
time.sleep(5)
print "%s (%s) is up." % (name, ip)
def launch_config_server():
# XXX: this doesn't perform a complete setup yet.
name = vps_util.new_vps_name('cs')
d = do_util.create_vps(name,
plan=vps_util.dc_by_cm(vps_util.my_cm()) + '_2GB')
if not vps_util.highstate_pid(name):
print("Highstate not running yet; waiting for a bit just in case...")
time.sleep(10)
while vps_util.highstate_pid(name):
print("Highstate still running...")
time.sleep(10)
ip = d['ip']
print "Checking for completion..."
while True:
try:
resp = requests.get(('http://%s/proxies.yaml.gz' % ip), timeout=5)
if resp.ok:
break
except requests.exceptions.RequestException:
pass
print "Server setup not complete yet; retrying..."
time.sleep(5)
print "%s (%s) is up." % (name, ip)
import multiprocessing
from Queue import Empty
import os
import time
import traceback
import yaml
import json
from alert import send_to_slack
from redis_util import redis_shell
import redisq
import vps_util
CM = vps_util.my_cm()
REGION = vps_util.my_region()
scope = os.environ['QSCOPE']
if scope == 'REGION':
QPREFIX = REGION
elif scope == 'CM':
QPREFIX = CM
else:
assert False
MAXPROCS = int(os.getenv('MAXPROCS'))
LAUNCH_TIMEOUT = 30 * 60
vps_shell = vps_util.vps_shell(CM)
def run():
