def provision(region, availability_zone, stack_name, cf_template_path,
launcher_ami, launcher_instance_type, worker_instance_type,
director_conf_path, cluster_ami, num_workers):
start_time = datetime.now()
# create cloudformation stack (VPC etc)
cf_conn = create_cf_connection(region)
create_cf_stack(cf_conn, stack_name, cf_template_path, availability_zone)
# create launcher instance
ec2_conn = create_ec2_connection(region)
launcher_instance = create_launcher_instance(
ec2_conn, cf_conn, stack_name, launcher_ami, launcher_instance_type)
# run bootstrap on launcher
execute(
run_director_bootstrap,
director_conf_path=director_conf_path, region=region,
cluster_ami=cluster_ami, num_workers=num_workers,
stack_name=stack_name, worker_instance_type=worker_instance_type,
hosts=[launcher_instance.ip_address])
end_time = datetime.now()
print "Cluster has started. Took {t} minutes.".format(
t=(end_time - start_time).seconds / 60)
def teardown(region, stack_name):
# terminate Hadoop cluster (prompts for confirmation)
ec2_conn = create_ec2_connection(region)
execute(run_director_terminate,
hosts=[get_launcher_instance(ec2_conn, stack_name).ip_address])
# terminate launcher instance
terminate_launcher_instance(ec2_conn, stack_name)
# delete stack
cf_conn = create_cf_connection(region)
delete_stack(cf_conn, stack_name)
def run_director_bootstrap(director_conf_path, region, cluster_ami,
num_workers, stack_name, worker_instance_type):
# replace variables in conf template and copy to launcher
cf_conn = create_cf_connection(region)
params = {'accessKeyId': get_aws_access_key_id(),
'secretAccessKey': get_aws_secret_access_key(),
'region': region,
'stack_name': stack_name,
'owner': getuser(),
'keyName': get_ec2_key_pair(),
'subnetId': get_subnet_id(cf_conn, stack_name),
'securityGroupsIds': get_security_group_id(cf_conn, stack_name),
'image': cluster_ami,
'num_workers': num_workers,
'worker_instance_type': worker_instance_type}
with open(director_conf_path, 'r') as template_file:
interpolated_body = template_file.read() % params
director_conf = StringIO(interpolated_body)
put(director_conf, 'director.conf')
# bootstrap the Hadoop cluster
run('cloudera-director bootstrap director.conf')
