def __init__(self, config=None): if config: config = defaultdict(str, config) for k in Cluster.keys: setattr(self, k, config[k]) self.sdb = boto.connect_sdb(ACCESS_KEY_ID, SECRET_ACCESS_KEY) self.domain = util.getAWS(self.sdb, "clusters")
def add_instance(self): image = EC2MPI.ec2.get_image(self.ami) util.getAWS(EC2MPI.s3, self.mpi_bucket) bucket = util.getAWS(EC2MPI.s3, self.s3_bucket) pri_key = bucket.get_key("id-%s" % self.mpi_key) pub_key = bucket.get_key("id-%s.pub" % self.mpi_key) pri_key_url = pri_key.generate_url(300) pub_key_url = pub_key.generate_url(300) startup_script_dict = { "access_key": ACCESS_KEY_ID, "secret_key": SECRET_ACCESS_KEY, "bucket": self.s3_bucket, "pri_key_url": pri_key_url, "pub_key_url": pub_key_url, "mpi_bucket": self.mpi_bucket, } print "Starting up new instance" reservation = image.run( min_count=1, max_count=1, key_name=AWS_KEYPAIR_NAME, instance_type=self.instance_type, placement=self.placement, user_data=EC2MPI.startup_script % startup_script_dict, ) print reservation while 1: if reservation.instances[0].update() == u"running": break else: time.sleep(2) self.instances += [reservation.instances[0].id] self.count = int(self.count) + 1 self.save() print "Instance is ready" self.update_machinefile()
def start_cluster(config=None): # See if user is passing in a config if not config: config = EC2MPI.run_config() cluster = Cluster(config) util.getAWS(EC2MPI.s3, config["mpi_bucket"]) bucket = util.getAWS(EC2MPI.s3, config["s3_bucket"]) pri_key = bucket.get_key("id-%s" % config["mpi_key"]) pub_key = bucket.get_key("id-%s.pub" % config["mpi_key"]) pri_key_url = pri_key.generate_url(300) pub_key_url = pub_key.generate_url(300) startup_script_dict = { "access_key": ACCESS_KEY_ID, "secret_key": SECRET_ACCESS_KEY, "bucket": config["s3_bucket"], "pri_key_url": pri_key_url, "pub_key_url": pub_key_url, "mpi_bucket": config["mpi_bucket"], } # Startup the instances print "Starting up" reservation = EC2MPI.ec2.run_instances( image_id=config["ami"], min_count=cluster.min_count, max_count=cluster.max_count, key_name=AWS_KEYPAIR_NAME, instance_type=cluster.instance_type, placement=cluster.placement, user_data=EC2MPI.startup_script % startup_script_dict, ) cluster.reservation = reservation.id cluster.status = "starting" cluster.save() # Wait for instances to all be available print "Waiting for instances" instances = reservation.instances while 1: for instance in instances: if instance.update() == u"running": print "Instance %s is running" % instance.id instances.remove(instance) if len(instances) == 0: print "All done" break else: time.sleep(2) reservation = EC2MPI.get_reservation(cluster.reservation) count = 0 instance_ids = [] for instance in reservation.instances: if instance.update() == u"running": count += 1 instance_ids += [instance.id] print instance.update(), instance.ami_launch_index, instance.public_dns_name cluster.instances = instance_ids cluster.count = count cluster.status = "pending" cluster.save() # Build machinefile and upload to master node cluster.update_machinefile() print "Cluster is up and ready" cluster.hostname = master_node.public_dns_name cluster.status = "running" cluster.save() return cluster