def __init__(self, config=None):
if config:
config = defaultdict(str, config)
for k in Cluster.keys:
setattr(self, k, config[k])
self.sdb = boto.connect_sdb(ACCESS_KEY_ID, SECRET_ACCESS_KEY)
self.domain = util.getAWS(self.sdb, "clusters")
def add_instance(self):
image = EC2MPI.ec2.get_image(self.ami)
util.getAWS(EC2MPI.s3, self.mpi_bucket)
bucket = util.getAWS(EC2MPI.s3, self.s3_bucket)
pri_key = bucket.get_key("id-%s" % self.mpi_key)
pub_key = bucket.get_key("id-%s.pub" % self.mpi_key)
pri_key_url = pri_key.generate_url(300)
pub_key_url = pub_key.generate_url(300)
startup_script_dict = {
"access_key": ACCESS_KEY_ID,
"secret_key": SECRET_ACCESS_KEY,
"bucket": self.s3_bucket,
"pri_key_url": pri_key_url,
"pub_key_url": pub_key_url,
"mpi_bucket": self.mpi_bucket,
}
print "Starting up new instance"
reservation = image.run(
min_count=1,
max_count=1,
key_name=AWS_KEYPAIR_NAME,
instance_type=self.instance_type,
placement=self.placement,
user_data=EC2MPI.startup_script % startup_script_dict,
)
print reservation
while 1:
if reservation.instances[0].update() == u"running":
break
else:
time.sleep(2)
self.instances += [reservation.instances[0].id]
self.count = int(self.count) + 1
self.save()
print "Instance is ready"
self.update_machinefile()
def start_cluster(config=None):
# See if user is passing in a config
if not config:
config = EC2MPI.run_config()
cluster = Cluster(config)
util.getAWS(EC2MPI.s3, config["mpi_bucket"])
bucket = util.getAWS(EC2MPI.s3, config["s3_bucket"])
pri_key = bucket.get_key("id-%s" % config["mpi_key"])
pub_key = bucket.get_key("id-%s.pub" % config["mpi_key"])
pri_key_url = pri_key.generate_url(300)
pub_key_url = pub_key.generate_url(300)
startup_script_dict = {
"access_key": ACCESS_KEY_ID,
"secret_key": SECRET_ACCESS_KEY,
"bucket": config["s3_bucket"],
"pri_key_url": pri_key_url,
"pub_key_url": pub_key_url,
"mpi_bucket": config["mpi_bucket"],
}
# Startup the instances
print "Starting up"
reservation = EC2MPI.ec2.run_instances(
image_id=config["ami"],
min_count=cluster.min_count,
max_count=cluster.max_count,
key_name=AWS_KEYPAIR_NAME,
instance_type=cluster.instance_type,
placement=cluster.placement,
user_data=EC2MPI.startup_script % startup_script_dict,
)
cluster.reservation = reservation.id
cluster.status = "starting"
cluster.save()
# Wait for instances to all be available
print "Waiting for instances"
instances = reservation.instances
while 1:
for instance in instances:
if instance.update() == u"running":
print "Instance %s is running" % instance.id
instances.remove(instance)
if len(instances) == 0:
print "All done"
break
else:
time.sleep(2)
reservation = EC2MPI.get_reservation(cluster.reservation)
count = 0
instance_ids = []
for instance in reservation.instances:
if instance.update() == u"running":
count += 1
instance_ids += [instance.id]
print instance.update(), instance.ami_launch_index, instance.public_dns_name
cluster.instances = instance_ids
cluster.count = count
cluster.status = "pending"
cluster.save()
# Build machinefile and upload to master node
cluster.update_machinefile()
print "Cluster is up and ready"
cluster.hostname = master_node.public_dns_name
cluster.status = "running"
cluster.save()
return cluster
