def all_vpss(provider):
print "Fetching VPS list for provider '%s'..." % provider
ret = vps_util.vps_shell(provider).all_vpss()
print "...done fetching VPS list for provider '%s'." % provider
print
return ret
if local_version == remote_version:
return set(map(str.strip, file(cm + '_vpss')))
else:
ret = redis_shell.lrange(cm + ':vpss', 0, -1)
file(cm + '_vpss', 'w').write('\n'.join(ret))
file(cm + '_vpss_version', 'w').write(remote_version)
return set(ret)
def in_production(name):
return (not name.startswith('fp-')
or vps_util.cm_by_name(name) in ['doams3', 'dosgp1'])
expected_do = vpss_from_cm("doams3") | vpss_from_cm('dosgp1')
expected_vultr = vpss_from_cm("vltok1")
actual_do = set(v.name for v in vps_util.vps_shell('do').all_vpss()
if in_production(v.name))
actual_vultr = set(v.name for v in vps_util.vps_shell('vl').all_vpss())
errors = []
for caption, vpss in [("Missing DO droplets", expected_do - actual_do),
("Unexpected DO droplets", actual_do - expected_do),
("Missing Vultr VPSs", expected_vultr - actual_vultr),
("Unexpected Vultr VPSs", actual_vultr - expected_vultr)]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
def send_mail(from_, to, subject, body):
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = from_
if not entry.startswith('-1')
]))
dcs = {
'do': ['doams3', 'dosgp1', 'donyc3', 'dosfo1'],
'li': ['lisgp1', 'litok1']
}
expected = {
provider: set.union(*map(vpss_from_cm, provider_dcs))
for provider, provider_dcs in dcs.iteritems()
}
actual = {
provider: set(v.name for v in vps_util.vps_shell(provider).all_vpss()
if not v.name.startswith('fp-')
or vps_util.is_production_proxy(v.name))
for provider in dcs
}
errors = []
for provider in dcs:
for caption, vpss in [(("Missing %s droplets" % provider.upper()),
expected[provider] - actual[provider]),
(("Unexpected %s droplets" % provider.upper()),
actual[provider] - expected[provider])]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
if errors:
except IOError:
local_version = None
remote_version = redis_shell.get(cm + ":vpss:version")
if local_version == remote_version:
return set(map(str.strip, file(cm + "_vpss")))
else:
ret = redis_shell.lrange(cm + ":vpss", 0, -1)
file(cm + "_vpss", "w").write("\n".join(ret))
file(cm + "_vpss_version", "w").write(remote_version)
return set(ret)
expected_do = vpss_from_cm("doams3") | vpss_from_cm("dosgp1")
expected_vultr = vpss_from_cm("vltok1")
actual_do = set(v.name for v in vps_util.vps_shell("do").all_vpss())
actual_vultr = set(v.name for v in vps_util.vps_shell("vl").all_vpss())
errors = []
for caption, vpss in [
("Missing DO droplets", expected_do - actual_do),
("Unexpected DO droplets", actual_do - expected_do),
("Missing Vultr VPSs", expected_vultr - actual_vultr),
("Unexpected Vultr VPSs", actual_vultr - expected_vultr),
]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
def send_mail(from_, to, subject, body):
msg = MIMEText(body)
return set(map(str.strip, file(cm + '_vpss')))
else:
ret = redis_shell.lrange(cm + ':vpss', 0, -1)
file(cm + '_vpss', 'w').write('\n'.join(ret))
file(cm + '_vpss_version', 'w').write(remote_version)
return set(ret)
dcs = {'do': ['doams3', 'dosgp1', 'donyc3', 'dosfo1'],
'vl': ['vltok1', 'vlfra1', 'vlpar1'],
'li': ['lisgp1', 'litok1']}
expected = {provider: set.union(*map(vpss_from_cm, provider_dcs))
for provider, provider_dcs in dcs.iteritems()}
actual = {provider: set(v.name
for v in vps_util.vps_shell(provider).all_vpss()
if not v.name.startswith('fp-')
or vps_util.is_production_proxy(v.name))
for provider in dcs}
errors = []
for provider in dcs:
for caption, vpss in [(("Missing %s droplets" % provider.upper()),
expected[provider] - actual[provider]),
(("Unexpected %s droplets" % provider.upper()),
actual[provider] - expected[provider])]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
if errors:
for error in errors:
import vps_util
CM = vps_util.my_cm()
REGION = vps_util.my_region()
scope = os.environ['QSCOPE']
if scope == 'REGION':
QPREFIX = REGION
elif scope == 'CM':
QPREFIX = CM
else:
assert False
MAXPROCS = int(os.getenv('MAXPROCS'))
LAUNCH_TIMEOUT = 30 * 60
vps_shell = vps_util.vps_shell(CM)
def run():
qname = QPREFIX + ":srvreqq"
print "Serving queue", qname, ", MAXPROCS:", repr(MAXPROCS)
quarantine = CM + ":quarantined_vpss"
reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT)
procq = multiprocessing.Queue()
pending = {}
def kill_task(reqid):
print "Killing timed out process and vps..."
task = pending.pop(reqid)
task['proc'].terminate()
proc = multiprocessing.Process(target=vps_shell.destroy_vps,
args=(task['name'],))
except IOError:
local_version = None
remote_version = redis_shell.get(cm + ':vpss:version')
if local_version == remote_version:
return set(map(str.strip, file(cm + '_vpss')))
else:
ret = redis_shell.lrange(cm + ':vpss', 0, -1)
file(cm + '_vpss', 'w').write('\n'.join(ret))
file(cm + '_vpss_version', 'w').write(remote_version)
return set(ret)
expected_do = vpss_from_cm("doams3") | vpss_from_cm('dosgp1')
expected_vultr = vpss_from_cm("vltok1")
actual_do = set(v.name for v in vps_util.vps_shell('do').all_vpss())
actual_vultr = set(v.name for v in vps_util.vps_shell('vl').all_vpss())
errors = []
for caption, vpss in [("Missing DO droplets", expected_do - actual_do),
("Unexpected DO droplets", actual_do - expected_do),
("Missing Vultr VPSs", expected_vultr - actual_vultr),
("Unexpected Vultr VPSs", actual_vultr - expected_vultr)
]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
def send_mail(from_, to, subject, body):
msg = MIMEText(body)
msg['Subject'] = subject
def all_vpss(provider):
print "Fetching VPS list for provider '%s'..." % provider
ret = vps_util.vps_shell(provider).all_vpss()
print "...done fetching VPS list for provider '%s'." % provider
print
return ret
def vpss(provider_etc):
return vps_util.vps_shell(provider_etc).all_vpss()
def all_vpss():
return [
v for provider in ['vl', 'do']
for v in vps_util.vps_shell(provider).all_vpss()
]
def vpss(provider_etc):
return vps_util.vps_shell(provider_etc).all_vpss()
def all_vpss():
return [v
for provider in ['vl', 'do']
for v in vps_util.vps_shell(provider).all_vpss()]
if local_version == remote_version:
return set(map(str.strip, file(cm + '_vpss')))
else:
ret = redis_shell.lrange(cm + ':vpss', 0, -1)
file(cm + '_vpss', 'w').write('\n'.join(ret))
file(cm + '_vpss_version', 'w').write(remote_version)
return set(ret)
expected_do = vpss_from_cm('doams3') | vpss_from_cm('dosgp1') | vpss_from_cm(
'donyc3') | vpss_from_cm('dosfo1')
expected_vultr = vpss_from_cm('vltok1') | vpss_from_cm(
'vlfra1') | vpss_from_cm('vlpar1')
actual_do = set(
v.name for v in vps_util.vps_shell('do').all_vpss()
if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name))
actual_vultr = set(
v.name for v in vps_util.vps_shell('vl').all_vpss()
if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name))
errors = []
for caption, vpss in [("Missing DO droplets", expected_do - actual_do),
("Unexpected DO droplets", actual_do - expected_do),
("Missing Vultr VPSs", expected_vultr - actual_vultr),
("Unexpected Vultr VPSs", actual_vultr - expected_vultr)
]:
if vpss:
errors.append(caption + ": " + ", ".join(sorted(vpss)))
if errors:
