def all_vpss(provider): print "Fetching VPS list for provider '%s'..." % provider ret = vps_util.vps_shell(provider).all_vpss() print "...done fetching VPS list for provider '%s'." % provider print return ret
if local_version == remote_version: return set(map(str.strip, file(cm + '_vpss'))) else: ret = redis_shell.lrange(cm + ':vpss', 0, -1) file(cm + '_vpss', 'w').write('\n'.join(ret)) file(cm + '_vpss_version', 'w').write(remote_version) return set(ret) def in_production(name): return (not name.startswith('fp-') or vps_util.cm_by_name(name) in ['doams3', 'dosgp1']) expected_do = vpss_from_cm("doams3") | vpss_from_cm('dosgp1') expected_vultr = vpss_from_cm("vltok1") actual_do = set(v.name for v in vps_util.vps_shell('do').all_vpss() if in_production(v.name)) actual_vultr = set(v.name for v in vps_util.vps_shell('vl').all_vpss()) errors = [] for caption, vpss in [("Missing DO droplets", expected_do - actual_do), ("Unexpected DO droplets", actual_do - expected_do), ("Missing Vultr VPSs", expected_vultr - actual_vultr), ("Unexpected Vultr VPSs", actual_vultr - expected_vultr)]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) def send_mail(from_, to, subject, body): msg = MIMEText(body) msg['Subject'] = subject msg['From'] = from_
if not entry.startswith('-1') ])) dcs = { 'do': ['doams3', 'dosgp1', 'donyc3', 'dosfo1'], 'li': ['lisgp1', 'litok1'] } expected = { provider: set.union(*map(vpss_from_cm, provider_dcs)) for provider, provider_dcs in dcs.iteritems() } actual = { provider: set(v.name for v in vps_util.vps_shell(provider).all_vpss() if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name)) for provider in dcs } errors = [] for provider in dcs: for caption, vpss in [(("Missing %s droplets" % provider.upper()), expected[provider] - actual[provider]), (("Unexpected %s droplets" % provider.upper()), actual[provider] - expected[provider])]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) if errors:
except IOError: local_version = None remote_version = redis_shell.get(cm + ":vpss:version") if local_version == remote_version: return set(map(str.strip, file(cm + "_vpss"))) else: ret = redis_shell.lrange(cm + ":vpss", 0, -1) file(cm + "_vpss", "w").write("\n".join(ret)) file(cm + "_vpss_version", "w").write(remote_version) return set(ret) expected_do = vpss_from_cm("doams3") | vpss_from_cm("dosgp1") expected_vultr = vpss_from_cm("vltok1") actual_do = set(v.name for v in vps_util.vps_shell("do").all_vpss()) actual_vultr = set(v.name for v in vps_util.vps_shell("vl").all_vpss()) errors = [] for caption, vpss in [ ("Missing DO droplets", expected_do - actual_do), ("Unexpected DO droplets", actual_do - expected_do), ("Missing Vultr VPSs", expected_vultr - actual_vultr), ("Unexpected Vultr VPSs", actual_vultr - expected_vultr), ]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) def send_mail(from_, to, subject, body): msg = MIMEText(body)
return set(map(str.strip, file(cm + '_vpss'))) else: ret = redis_shell.lrange(cm + ':vpss', 0, -1) file(cm + '_vpss', 'w').write('\n'.join(ret)) file(cm + '_vpss_version', 'w').write(remote_version) return set(ret) dcs = {'do': ['doams3', 'dosgp1', 'donyc3', 'dosfo1'], 'vl': ['vltok1', 'vlfra1', 'vlpar1'], 'li': ['lisgp1', 'litok1']} expected = {provider: set.union(*map(vpss_from_cm, provider_dcs)) for provider, provider_dcs in dcs.iteritems()} actual = {provider: set(v.name for v in vps_util.vps_shell(provider).all_vpss() if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name)) for provider in dcs} errors = [] for provider in dcs: for caption, vpss in [(("Missing %s droplets" % provider.upper()), expected[provider] - actual[provider]), (("Unexpected %s droplets" % provider.upper()), actual[provider] - expected[provider])]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) if errors: for error in errors:
import vps_util CM = vps_util.my_cm() REGION = vps_util.my_region() scope = os.environ['QSCOPE'] if scope == 'REGION': QPREFIX = REGION elif scope == 'CM': QPREFIX = CM else: assert False MAXPROCS = int(os.getenv('MAXPROCS')) LAUNCH_TIMEOUT = 30 * 60 vps_shell = vps_util.vps_shell(CM) def run(): qname = QPREFIX + ":srvreqq" print "Serving queue", qname, ", MAXPROCS:", repr(MAXPROCS) quarantine = CM + ":quarantined_vpss" reqq = redisq.Queue(qname, redis_shell, LAUNCH_TIMEOUT) procq = multiprocessing.Queue() pending = {} def kill_task(reqid): print "Killing timed out process and vps..." task = pending.pop(reqid) task['proc'].terminate() proc = multiprocessing.Process(target=vps_shell.destroy_vps, args=(task['name'],))
except IOError: local_version = None remote_version = redis_shell.get(cm + ':vpss:version') if local_version == remote_version: return set(map(str.strip, file(cm + '_vpss'))) else: ret = redis_shell.lrange(cm + ':vpss', 0, -1) file(cm + '_vpss', 'w').write('\n'.join(ret)) file(cm + '_vpss_version', 'w').write(remote_version) return set(ret) expected_do = vpss_from_cm("doams3") | vpss_from_cm('dosgp1') expected_vultr = vpss_from_cm("vltok1") actual_do = set(v.name for v in vps_util.vps_shell('do').all_vpss()) actual_vultr = set(v.name for v in vps_util.vps_shell('vl').all_vpss()) errors = [] for caption, vpss in [("Missing DO droplets", expected_do - actual_do), ("Unexpected DO droplets", actual_do - expected_do), ("Missing Vultr VPSs", expected_vultr - actual_vultr), ("Unexpected Vultr VPSs", actual_vultr - expected_vultr) ]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) def send_mail(from_, to, subject, body): msg = MIMEText(body) msg['Subject'] = subject
def vpss(provider_etc): return vps_util.vps_shell(provider_etc).all_vpss()
def all_vpss(): return [ v for provider in ['vl', 'do'] for v in vps_util.vps_shell(provider).all_vpss() ]
def all_vpss(): return [v for provider in ['vl', 'do'] for v in vps_util.vps_shell(provider).all_vpss()]
if local_version == remote_version: return set(map(str.strip, file(cm + '_vpss'))) else: ret = redis_shell.lrange(cm + ':vpss', 0, -1) file(cm + '_vpss', 'w').write('\n'.join(ret)) file(cm + '_vpss_version', 'w').write(remote_version) return set(ret) expected_do = vpss_from_cm('doams3') | vpss_from_cm('dosgp1') | vpss_from_cm( 'donyc3') | vpss_from_cm('dosfo1') expected_vultr = vpss_from_cm('vltok1') | vpss_from_cm( 'vlfra1') | vpss_from_cm('vlpar1') actual_do = set( v.name for v in vps_util.vps_shell('do').all_vpss() if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name)) actual_vultr = set( v.name for v in vps_util.vps_shell('vl').all_vpss() if not v.name.startswith('fp-') or vps_util.is_production_proxy(v.name)) errors = [] for caption, vpss in [("Missing DO droplets", expected_do - actual_do), ("Unexpected DO droplets", actual_do - expected_do), ("Missing Vultr VPSs", expected_vultr - actual_vultr), ("Unexpected Vultr VPSs", actual_vultr - expected_vultr) ]: if vpss: errors.append(caption + ": " + ", ".join(sorted(vpss))) if errors: