forked from compiler-explorer/infra
-
Notifications
You must be signed in to change notification settings - Fork 0
/
update_instances.py
executable file
·75 lines (66 loc) · 2.74 KB
/
update_instances.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
#!/usr/bin/env python
import boto.ec2
import boto.ec2.autoscale
import boto.ec2.elb
from boto.manage.cmdshell import sshclient_from_instance
import time
def get_gcc_ex_group():
conn = boto.ec2.autoscale.connect_to_region("us-east-1")
all_groups = conn.get_all_groups(names=["Gcc Explorer"])
return all_groups[0]
def ensure_at_least_two():
print "ensuring at least two instances"
gcc_explorer_group = get_gcc_ex_group()
prev_min = gcc_explorer_group.min_size
print "Found {} instances".format(prev_min)
if prev_min < 2:
print "Updating min instances to 2"
gcc_explorer_group.min_size = 2
gcc_explorer_group.desired_capacity = 2
gcc_explorer_group.update()
return prev_min
def await_at_least_two_healthy():
print "Waiting for at least two healthy instances"
elb = boto.ec2.elb.connect_to_region('us-east-1')
balancer = elb.get_all_load_balancers(load_balancer_names=['GccExplorerVpc'])[0]
while True:
healthy = [s for s in balancer.get_instance_health() if s.state == u'InService']
if len(healthy) >= 2:
print "Found {} healthy".format(len(healthy))
break
print "Only {} healthy...waiting".format(len(healthy))
time.sleep(5)
print "Enough healthy instances"
def set_back_to(prev):
print "Putting back the number of instances to {}".format(prev)
g = get_gcc_ex_group()
g.min_size = prev
g.update()
def update_gcc_explorers():
prev = ensure_at_least_two()
await_at_least_two_healthy()
if prev != 1:
conn = boto.ec2.connect_to_region('us-east-1')
reservations = conn.get_all_instances()
for reservation in reservations:
for instance in reservation.instances:
if instance.state != 'running':
print "Skipping {} instance {}".format(instance.state, instance.id)
continue
if "App" not in instance.tags or instance.tags["App"] != "GccExplorer":
print "Skipping non-gcc explorer instance {}".format(instance.id)
continue
print "Connecting to", instance
ssh_client = sshclient_from_instance(instance, "ec2-mattgodbolt.pem",
user_name='ubuntu')
print "Connected. Running command"
status, stdout, stderr = ssh_client.run('sudo -i docker pull -a mattgodbolt/gcc-explorer && sudo service gcc-explorer restart')
print "Status", status
print "Stdout", stdout
print "Stderr", stderr
print "Done, waiting a minute"
time.sleep(60)
await_at_least_two_healthy()
set_back_to(prev)
if __name__ == '__main__':
update_gcc_explorers()