def startup_instances(num, cluster, type, roletag): global running, newmachines if num == 0: return [] machines = int(math.ceil(num / float(cluster))) running_ = ec2.get_running_instances(type, ['role', roletag]) pending_ = [] if len(running_) < machines: running_ += ec2.get_running_instances(type, ['role', 'none']) if len(running_) < machines: print "Adding machines for role: " + roletag nstarted = len(running_) stopped_ = ec2.get_stopped_instances(type) if stopped_: ntostart = min(len(stopped_), machines - nstarted) ec2.startup_machines(stopped_[0:ntostart]) pending_ += stopped_[0:ntostart] nstarted += ntostart if nstarted < machines: if ondemand: checkedout = ec2.checkout_machines(machines - nstarted, type) else: requests = ec2.request_machines(machines - nstarted, type) rids = [r.id for r in requests] while (True): print "Waiting for spot instance fulfillment..." open = False requests = ec2.update_spot_requests(rids) if len(requests) != len(rids): print "Problem updating spot request state!" kill() for r in requests: if r.state == 'active': continue elif r.state == 'open': if r.status == 'price-too-low': print "You bid too low!" kill() open = True else: print "Bad spot request state: (" + r.id + ", " + r.state + ")" kill() if not open: break sleep(10) checkedout = ec2.get_instances( [r.instance_id for r in requests]) pending_ += checkedout newmachines += checkedout else: print "Using existing machines for role: " + roletag while (pending_): print "Waiting for machines to start..." again = [] for p in pending_: if ec2.is_running(p): p.add_tag('role', roletag) else: again.append(p) pending_ = again sleep(5) hosts_ = [] running_ = ec2.get_running_instances(type, ['role', roletag]) + \ ec2.get_running_instances(type, ['role', 'none']) running_ = running_[0:machines] running += running_ for r in running_: r.update(True) r.add_tag('role', roletag) return running_
def startup_instances(num, cluster, type, roletag): global running, newmachines if num == 0: return [] machines = int(math.ceil(num / float(cluster))) running_ = ec2.get_running_instances(type, ['role', roletag]) pending_ = [] if len(running_) < machines: running_ += ec2.get_running_instances(type, ['role', 'none']) if len(running_) < machines: print "Adding machines for role: " + roletag nstarted = len(running_) stopped_ = ec2.get_stopped_instances(type) if stopped_: ntostart = min(len(stopped_), machines - nstarted) ec2.startup_machines(stopped_[0:ntostart]) pending_ += stopped_[0:ntostart] nstarted += ntostart if nstarted < machines: if ondemand: checkedout = ec2.checkout_machines(machines - nstarted, type) else: requests = ec2.request_machines(machines - nstarted, type) rids = [r.id for r in requests] while(True): print "Waiting for spot instance fulfillment..." open = False requests = ec2.update_spot_requests(rids) if len(requests) != len(rids): print "Problem updating spot request state!" kill() for r in requests: if r.state == 'active': continue elif r.state == 'open': if r.status == 'price-too-low': print "You bid too low!" kill() open = True else: print "Bad spot request state: (" + r.id + ", " + r.state + ")" kill() if not open: break sleep(10) checkedout = ec2.get_instances([r.instance_id for r in requests]) pending_ += checkedout newmachines += checkedout else: print "Using existing machines for role: " + roletag while (pending_): print "Waiting for machines to start..." again = [] for p in pending_: if ec2.is_running(p): p.add_tag('role', roletag) else: again.append(p) pending_ = again sleep(5) hosts_ = [] running_ = ec2.get_running_instances(type, ['role', roletag]) + \ ec2.get_running_instances(type, ['role', 'none']) running_ = running_[0:machines] running += running_ for r in running_: r.update(True) r.add_tag('role', roletag) return running_
def kill(): print "Going nuclear on EC2" ec2.terminate_machines( [i for i in ec2.get_all_instances() if 'hold' not in i.tags]) ec2.cancel_spot_requests(ec2.get_all_spot_requests()) exit(0) ec2.connect() if justkill: kill() if invoke or invokebg: running = ec2.get_running_instances( tag=['role', filter] if filter else None) if invokebg: procs = [ ec2.run_ssh_command_bg(r.public_dns_name, invokebg, tty=sshtty) for r in running ] for p in procs: p.wait() else: [ ec2.run_ssh_command(r.public_dns_name, invoke, tty=sshtty) for r in running ] exit(0)
exec(exph, globals()) exph.close() def kill(): print "Going nuclear on EC2" ec2.terminate_machines([i for i in ec2.get_all_instances() if 'hold' not in i.tags]) ec2.cancel_spot_requests(ec2.get_all_spot_requests()) exit(0) ec2.connect() if justkill: kill() if invoke or invokebg: running = ec2.get_running_instances(tag=['role', filter] if filter else None) if invokebg: procs = [ec2.run_ssh_command_bg(r.public_dns_name, invokebg, tty=sshtty) for r in running] for p in procs: p.wait() else: [ec2.run_ssh_command(r.public_dns_name, invoke, tty=sshtty) for r in running] exit(0) running = [] newmachines = [] def startup_instances(num, cluster, type, roletag): global running, newmachines