def request_spot_instances(*args, **kwargs): """ec2 nodes request spot instances name_or_config price Request spot instances to be used in this cluster. Unlike boot, this doesn't block because we can't be sure the nodes will actually be booted immediately. You should use this in conjunction with the import nodes command once the nodes have booted, which will complete the setup. The main benefit of using this instead of allocating the nodes manually is that all the configuration is setup properly, including, importantly, the initial script which performs the bootstrapping configuration. """ name_or_config, price = arguments.parse_or_die(request_spot_instances, [object, str], *args) name, cc = name_and_config(name_or_config) if 'reservation' in cc.state or 'spot' in cc.state or 'instances' in cc.state: print "It looks like you already have active nodes for this cluster..." exit(1) # Load the setup script template, replace puppet master info user_data = data.load('ec2-user-data', 'node-setup.sh') user_data = user_data.replace('{{{PUPPET_MASTER}}}', cc.puppet_master) # Now create the nodes conn = EC2Connection(config.AWS_ACCESS_KEY_ID, config.AWS_SECRET_ACCESS_KEY) request = conn.request_spot_instances(price, cc.ami, # launch group is just a # name that causes these to # only launch if all can be # satisfied launch_group=name, # availability zone group # lets use specify a group # name such that we'll group # all instances together availability_zone_group=(name+'_azg'), count=cc.size, key_name=cc.keypair, instance_type=cc.instance_type, security_groups=[cc.group], user_data=user_data ) # Indicate that we've done a spot request so we don't try to double-allocate nodes. cc.state['spot'] = True cc.save() print "Requested %d spot instances" % cc.size return 0
def boot(*args, **kwargs): """ec2 nodes boot name_or_config [--wait-timeout=300 --pem=/path/to/key.pem] Boot a cluster's nodes. The command will block for wait-timeout seconds, or until all nodes reach a ready state (currently defined as being pingable and containing files indicating readiness. A wait-timeout of 0 disables this. A pem file, either passed on the command line or through the environment is required for the timeout to work properly. Note that with timeouts enabled, this will check that the nodes reach a ready state. """ name_or_config = arguments.parse_or_die(boot, [object], *args) timeout = config.kwarg_or_default('wait-timeout', kwargs, default=600) # Note pemfile is different from other places since it's only required with wait-timeout. pemfile = config.kwarg_or_get('pem', kwargs, 'SIRIKATA_CLUSTER_PEMFILE', default=None) name, cc = name_and_config(name_or_config) if 'reservation' in cc.state or 'spot' in cc.state or 'instances' in cc.state: print "It looks like you already have active nodes for this cluster..." exit(1) if timeout > 0 and not pemfile: print "You need to specify a pem file to use timeouts." exit(1) # Load the setup script template, replace puppet master info user_data = data.load('ec2-user-data', 'node-setup.sh') user_data = user_data.replace('{{{PUPPET_MASTER}}}', cc.puppet_master) # Unlike spot instances, where we can easily request that any # availability zone be used by that all be in the same AZ, here we # have to specify an AZ directly. We just choose one randomly for now... conn = EC2Connection(config.AWS_ACCESS_KEY_ID, config.AWS_SECRET_ACCESS_KEY) zones = conn.get_all_zones() zone = random.choice(zones).name # Now create the nodes reservation = conn.run_instances(cc.ami, placement=zone, min_count=cc.size, max_count=cc.size, key_name=cc.keypair, instance_type=cc.instance_type, security_groups=[cc.group], user_data=user_data ) # Save reservation, instance info cc.state['reservation'] = reservation.id cc.state['instances'] = [inst.id for inst in reservation.instances] cc.save() # Cache some information about the instances which shouldn't # change. However, this can take some time to come up properly, so # we may need to poll a few times before we get the right info print "Collecting node information..." while True: new_instances = get_all_instances(cc, conn) if any([inst.ip_address is None or inst.dns_name is None or inst.private_ip_address is None or inst.private_dns_name is None for inst in new_instances.values()]): time.sleep(5) continue cc.state['instance_props'] = dict( [ (inst.id, { 'id' : inst.id, 'ip' : inst.ip_address, 'hostname' : inst.dns_name, 'private_ip' : inst.private_ip_address, 'private_hostname' : inst.private_dns_name, }) for inst in new_instances.values()]) break cc.save() return name_and_boot_nodes(cc, conn, pemfile, timeout)