def test_get_buildslave_instances(conn, example_instances):
e = example_instances
conn.return_value.get_only_instances.return_value = e
assert get_buildslave_instances("r", ["m1", "m2"]) == [e[0], e[1], e[2]]
conn.return_value.get_only_instances.assert_called_once_with(
filters={'tag:moz-state': 'ready',
'instance-state-name': 'running'})
def test_get_buildslave_instances(conn, example_instances):
e = example_instances
conn.return_value.get_only_instances.return_value = e
assert get_buildslave_instances("r", ["m1", "m2"]) == [e[0], e[1], e[2]]
conn.return_value.get_only_instances.assert_called_once_with(
filters={'tag:moz-state': 'ready',
'instance-state-name': 'running'})
def aws_stop_idle(user, key_filename, regions, masters_json, moz_types,
dryrun=False, concurrency=8):
if not regions:
# Look at all regions
log.debug("loading all regions")
regions = [r.name for r in boto.ec2.regions()]
min_running_by_type = 0
all_instances = []
impaired_ids = []
for r in regions:
log.debug("looking at region %s", r)
instances = get_buildslave_instances(r, moz_types)
log.debug("Got %s buildslave instances", len(instances))
impaired_ids.extend(get_impaired_instance_ids(r))
log.debug("Got %s impaired instances", len(impaired_ids))
instances_by_type = {}
for i in instances:
# TODO: Check if launch_time is too old, and terminate the instance
# if it is
# NB can't turn this on until aws_create_instance is working
# properly (with ssh keys)
instances_by_type.setdefault(i.tags['moz-type'], []).append(i)
# Make sure min_running_by_type are kept running
for t in instances_by_type:
to_remove = instances_by_type[t][:min_running_by_type]
for i in to_remove:
log.debug("%s - keep running (min %s instances of type %s)",
i.tags['Name'], min_running_by_type,
i.tags['moz-type'])
instances.remove(i)
all_instances.extend(instances)
random.shuffle(all_instances)
q = Queue()
to_stop = Queue()
def worker():
while True:
try:
i = q.get(timeout=0.1)
except Empty:
return
try:
if aws_safe_stop_instance(i, impaired_ids, user, key_filename,
masters_json, dryrun=dryrun):
to_stop.put(i)
except Exception:
log.debug("%s - unable to stop" % i.tags.get('Name'),
exc_info=True)
for i in all_instances:
q.put(i)
# Workaround for http://bugs.python.org/issue11108
time.strptime("19000102030405", "%Y%m%d%H%M%S")
threads = []
for i in range(concurrency):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
while threads:
for t in threads[:]:
try:
if t.is_alive():
t.join(timeout=0.5)
else:
t.join()
threads.remove(t)
except KeyboardInterrupt:
raise SystemExit(1)
total_stopped = {}
while not to_stop.empty():
i = to_stop.get()
if not dryrun:
i.update()
if 'moz-type' not in i.tags:
log.debug("%s - has no moz-type! (%s)" % (i.tags.get('Name'),
i.id))
t = i.tags.get('moz-type', 'none')
if t not in total_stopped:
total_stopped[t] = 0
total_stopped[t] += 1
for t, c in sorted(total_stopped.items()):
log.debug("%s - stopped %s", t, c)
gr_log.add("stopped.%s" % t, c)
def aws_stop_idle(user,
key_filename,
regions,
masters_json,
moz_types,
dryrun=False,
concurrency=8):
if not regions:
# Look at all regions
log.debug("loading all regions")
regions = [r.name for r in boto.ec2.regions()]
min_running_by_type = 0
all_instances = []
impaired_ids = []
for r in regions:
log.debug("looking at region %s", r)
instances = get_buildslave_instances(r, moz_types)
log.debug("Got %s buildslave instances", len(instances))
impaired_ids.extend(get_impaired_instance_ids(r))
log.debug("Got %s impaired instances", len(impaired_ids))
instances_by_type = {}
for i in instances:
# TODO: Check if launch_time is too old, and terminate the instance
# if it is
# NB can't turn this on until aws_create_instance is working
# properly (with ssh keys)
instances_by_type.setdefault(i.tags['moz-type'], []).append(i)
# Make sure min_running_by_type are kept running
for t in instances_by_type:
to_remove = instances_by_type[t][:min_running_by_type]
for i in to_remove:
log.debug("%s - keep running (min %s instances of type %s)",
i.tags['Name'], min_running_by_type,
i.tags['moz-type'])
instances.remove(i)
all_instances.extend(instances)
random.shuffle(all_instances)
q = Queue()
to_stop = Queue()
def worker():
while True:
try:
i = q.get(timeout=0.1)
except Empty:
return
try:
if aws_safe_stop_instance(i,
impaired_ids,
user,
key_filename,
masters_json,
dryrun=dryrun):
to_stop.put(i)
except Exception:
log.debug("%s - unable to stop" % i.tags.get('Name'),
exc_info=True)
for i in all_instances:
q.put(i)
# Workaround for http://bugs.python.org/issue11108
time.strptime("19000102030405", "%Y%m%d%H%M%S")
threads = []
for i in range(concurrency):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
while threads:
for t in threads[:]:
try:
if t.is_alive():
t.join(timeout=0.5)
else:
t.join()
threads.remove(t)
except KeyboardInterrupt:
raise SystemExit(1)
total_stopped = {}
while not to_stop.empty():
i = to_stop.get()
if not dryrun:
i.update()
if 'moz-type' not in i.tags:
log.debug("%s - has no moz-type! (%s)" %
(i.tags.get('Name'), i.id))
t = i.tags.get('moz-type', 'none')
if t not in total_stopped:
total_stopped[t] = 0
total_stopped[t] += 1
for t, c in sorted(total_stopped.items()):
log.debug("%s - stopped %s", t, c)
gr_log.add("stopped.%s" % t, c)
