def test_get_buildslave_instances(conn, example_instances): e = example_instances conn.return_value.get_only_instances.return_value = e assert get_buildslave_instances("r", ["m1", "m2"]) == [e[0], e[1], e[2]] conn.return_value.get_only_instances.assert_called_once_with( filters={'tag:moz-state': 'ready', 'instance-state-name': 'running'})
def aws_stop_idle(user, key_filename, regions, masters_json, moz_types, dryrun=False, concurrency=8): if not regions: # Look at all regions log.debug("loading all regions") regions = [r.name for r in boto.ec2.regions()] min_running_by_type = 0 all_instances = [] impaired_ids = [] for r in regions: log.debug("looking at region %s", r) instances = get_buildslave_instances(r, moz_types) log.debug("Got %s buildslave instances", len(instances)) impaired_ids.extend(get_impaired_instance_ids(r)) log.debug("Got %s impaired instances", len(impaired_ids)) instances_by_type = {} for i in instances: # TODO: Check if launch_time is too old, and terminate the instance # if it is # NB can't turn this on until aws_create_instance is working # properly (with ssh keys) instances_by_type.setdefault(i.tags['moz-type'], []).append(i) # Make sure min_running_by_type are kept running for t in instances_by_type: to_remove = instances_by_type[t][:min_running_by_type] for i in to_remove: log.debug("%s - keep running (min %s instances of type %s)", i.tags['Name'], min_running_by_type, i.tags['moz-type']) instances.remove(i) all_instances.extend(instances) random.shuffle(all_instances) q = Queue() to_stop = Queue() def worker(): while True: try: i = q.get(timeout=0.1) except Empty: return try: if aws_safe_stop_instance(i, impaired_ids, user, key_filename, masters_json, dryrun=dryrun): to_stop.put(i) except Exception: log.debug("%s - unable to stop" % i.tags.get('Name'), exc_info=True) for i in all_instances: q.put(i) # Workaround for http://bugs.python.org/issue11108 time.strptime("19000102030405", "%Y%m%d%H%M%S") threads = [] for i in range(concurrency): t = threading.Thread(target=worker) t.start() threads.append(t) while threads: for t in threads[:]: try: if t.is_alive(): t.join(timeout=0.5) else: t.join() threads.remove(t) except KeyboardInterrupt: raise SystemExit(1) total_stopped = {} while not to_stop.empty(): i = to_stop.get() if not dryrun: i.update() if 'moz-type' not in i.tags: log.debug("%s - has no moz-type! (%s)" % (i.tags.get('Name'), i.id)) t = i.tags.get('moz-type', 'none') if t not in total_stopped: total_stopped[t] = 0 total_stopped[t] += 1 for t, c in sorted(total_stopped.items()): log.debug("%s - stopped %s", t, c) gr_log.add("stopped.%s" % t, c)