def startup_instances(num, cluster, type, roletag):
global running, newmachines
if num == 0:
return []
machines = int(math.ceil(num / float(cluster)))
running_ = ec2.get_running_instances(type, ['role', roletag])
pending_ = []
if len(running_) < machines:
running_ += ec2.get_running_instances(type, ['role', 'none'])
if len(running_) < machines:
print "Adding machines for role: " + roletag
nstarted = len(running_)
stopped_ = ec2.get_stopped_instances(type)
if stopped_:
ntostart = min(len(stopped_), machines - nstarted)
ec2.startup_machines(stopped_[0:ntostart])
pending_ += stopped_[0:ntostart]
nstarted += ntostart
if nstarted < machines:
if ondemand:
checkedout = ec2.checkout_machines(machines - nstarted, type)
else:
requests = ec2.request_machines(machines - nstarted, type)
rids = [r.id for r in requests]
while (True):
print "Waiting for spot instance fulfillment..."
open = False
requests = ec2.update_spot_requests(rids)
if len(requests) != len(rids):
print "Problem updating spot request state!"
kill()
for r in requests:
if r.state == 'active':
continue
elif r.state == 'open':
if r.status == 'price-too-low':
print "You bid too low!"
kill()
open = True
else:
print "Bad spot request state: (" + r.id + ", " + r.state + ")"
kill()
if not open:
break
sleep(10)
checkedout = ec2.get_instances(
[r.instance_id for r in requests])
pending_ += checkedout
newmachines += checkedout
else:
print "Using existing machines for role: " + roletag
while (pending_):
print "Waiting for machines to start..."
again = []
for p in pending_:
if ec2.is_running(p):
p.add_tag('role', roletag)
else:
again.append(p)
pending_ = again
sleep(5)
hosts_ = []
running_ = ec2.get_running_instances(type, ['role', roletag]) + \
ec2.get_running_instances(type, ['role', 'none'])
running_ = running_[0:machines]
running += running_
for r in running_:
r.update(True)
r.add_tag('role', roletag)
return running_
def startup_instances(num, cluster, type, roletag):
global running, newmachines
if num == 0:
return []
machines = int(math.ceil(num / float(cluster)))
running_ = ec2.get_running_instances(type, ['role', roletag])
pending_ = []
if len(running_) < machines:
running_ += ec2.get_running_instances(type, ['role', 'none'])
if len(running_) < machines:
print "Adding machines for role: " + roletag
nstarted = len(running_)
stopped_ = ec2.get_stopped_instances(type)
if stopped_:
ntostart = min(len(stopped_), machines - nstarted)
ec2.startup_machines(stopped_[0:ntostart])
pending_ += stopped_[0:ntostart]
nstarted += ntostart
if nstarted < machines:
if ondemand:
checkedout = ec2.checkout_machines(machines - nstarted, type)
else:
requests = ec2.request_machines(machines - nstarted, type)
rids = [r.id for r in requests]
while(True):
print "Waiting for spot instance fulfillment..."
open = False
requests = ec2.update_spot_requests(rids)
if len(requests) != len(rids):
print "Problem updating spot request state!"
kill()
for r in requests:
if r.state == 'active':
continue
elif r.state == 'open':
if r.status == 'price-too-low':
print "You bid too low!"
kill()
open = True
else:
print "Bad spot request state: (" + r.id + ", " + r.state + ")"
kill()
if not open:
break
sleep(10)
checkedout = ec2.get_instances([r.instance_id for r in requests])
pending_ += checkedout
newmachines += checkedout
else:
print "Using existing machines for role: " + roletag
while (pending_):
print "Waiting for machines to start..."
again = []
for p in pending_:
if ec2.is_running(p):
p.add_tag('role', roletag)
else:
again.append(p)
pending_ = again
sleep(5)
hosts_ = []
running_ = ec2.get_running_instances(type, ['role', roletag]) + \
ec2.get_running_instances(type, ['role', 'none'])
running_ = running_[0:machines]
running += running_
for r in running_:
r.update(True)
r.add_tag('role', roletag)
return running_
def kill():
print "Going nuclear on EC2"
ec2.terminate_machines(
[i for i in ec2.get_all_instances() if 'hold' not in i.tags])
ec2.cancel_spot_requests(ec2.get_all_spot_requests())
exit(0)
ec2.connect()
if justkill:
kill()
if invoke or invokebg:
running = ec2.get_running_instances(
tag=['role', filter] if filter else None)
if invokebg:
procs = [
ec2.run_ssh_command_bg(r.public_dns_name, invokebg, tty=sshtty)
for r in running
]
for p in procs:
p.wait()
else:
[
ec2.run_ssh_command(r.public_dns_name, invoke, tty=sshtty)
for r in running
]
exit(0)
exec(exph, globals())
exph.close()
def kill():
print "Going nuclear on EC2"
ec2.terminate_machines([i for i in ec2.get_all_instances() if 'hold' not in i.tags])
ec2.cancel_spot_requests(ec2.get_all_spot_requests())
exit(0)
ec2.connect()
if justkill:
kill()
if invoke or invokebg:
running = ec2.get_running_instances(tag=['role', filter] if filter else None)
if invokebg:
procs = [ec2.run_ssh_command_bg(r.public_dns_name, invokebg, tty=sshtty) for r in running]
for p in procs:
p.wait()
else:
[ec2.run_ssh_command(r.public_dns_name, invoke, tty=sshtty) for r in running]
exit(0)
running = []
newmachines = []
def startup_instances(num, cluster, type, roletag):
global running, newmachines