def create_launcher_instance(ec2_conn, cf_conn, stack_name, launcher_ami,
launcher_instance_type):
launcher_instances = get_tagged_instances(
ec2_conn, {'eggo_stack_name': stack_name,
'eggo_node_type': 'launcher'})
if len(launcher_instances) > 0:
print "Launcher instance ({instance}) already exists. Reusing.".format(
instance=launcher_instances[0].ip_address)
return launcher_instances[0]
print "Creating launcher instance."
# see http://stackoverflow.com/questions/19029588/how-to-auto-assign-public-ip-to-ec2-instance-with-boto
interface = NetworkInterfaceSpecification(
subnet_id=get_subnet_id(cf_conn, stack_name),
groups=[get_security_group_id(cf_conn, stack_name)],
associate_public_ip_address=True)
interfaces = NetworkInterfaceCollection(interface)
reservation = ec2_conn.run_instances(
launcher_ami,
key_name=get_ec2_key_pair(),
instance_type=launcher_instance_type,
network_interfaces=interfaces)
launcher_instance = reservation.instances[0]
launcher_instance.add_tag('owner', getuser())
launcher_instance.add_tag('ec2_key_pair', get_ec2_key_pair())
launcher_instance.add_tag('eggo_stack_name', stack_name)
launcher_instance.add_tag('eggo_node_type', 'launcher')
wait_for_instance_state(ec2_conn, launcher_instance)
execute(install_director_client, hosts=[launcher_instance.ip_address])
execute(install_private_key, hosts=[launcher_instance.ip_address])
return launcher_instance
def create_cf_stack(cf_conn, stack_name, cf_template_path, availability_zone):
try:
if len(cf_conn.describe_stacks(stack_name)) > 0:
print "Stack '{n}' already exists. Reusing.".format(n=stack_name)
return
except BotoServerError:
# stack does not exist
pass
print "Creating stack with name '{n}'.".format(n=stack_name)
with open(cf_template_path, 'r') as template_file:
template_body=template_file.read()
cf_conn.create_stack(stack_name, template_body=template_body,
parameters=[('KeyPairName', get_ec2_key_pair()),
('AZ', availability_zone)],
tags={'owner': getuser(),
'ec2_key_pair': get_ec2_key_pair()})
wait_for_stack_status(cf_conn, stack_name, 'CREATE_COMPLETE')
def run_director_bootstrap(director_conf_path, region, cluster_ami,
num_workers, stack_name, worker_instance_type):
# replace variables in conf template and copy to launcher
cf_conn = create_cf_connection(region)
params = {'accessKeyId': get_aws_access_key_id(),
'secretAccessKey': get_aws_secret_access_key(),
'region': region,
'stack_name': stack_name,
'owner': getuser(),
'keyName': get_ec2_key_pair(),
'subnetId': get_subnet_id(cf_conn, stack_name),
'securityGroupsIds': get_security_group_id(cf_conn, stack_name),
'image': cluster_ami,
'num_workers': num_workers,
'worker_instance_type': worker_instance_type}
with open(director_conf_path, 'r') as template_file:
interpolated_body = template_file.read() % params
director_conf = StringIO(interpolated_body)
put(director_conf, 'director.conf')
# bootstrap the Hadoop cluster
run('cloudera-director bootstrap director.conf')
def run_director_bootstrap(director_conf_path, region, cluster_ami, num_workers, stack_name):
# replace variables in conf template and copy to launcher
cf_conn = create_cf_connection(region)
params = {
"accessKeyId": get_aws_access_key_id(),
"secretAccessKey": get_aws_secret_access_key(),
"region": region,
"stack_name": stack_name,
"owner": getuser(),
"keyName": get_ec2_key_pair(),
"subnetId": get_subnet_id(cf_conn, stack_name),
"securityGroupsIds": get_security_group_id(cf_conn, stack_name),
"image": cluster_ami,
"num_workers": num_workers,
}
with open(director_conf_path, "r") as template_file:
interpolated_body = template_file.read() % params
director_conf = StringIO(interpolated_body)
put(director_conf, "director.conf")
# bootstrap the Hadoop cluster
run("cloudera-director bootstrap director.conf")