def provision( region, availability_zone, stack_name, cf_template_path, launcher_ami, launcher_instance_type, director_conf_path, cluster_ami, num_workers, ): start_time = datetime.now() # create cloudformation stack (VPC etc) cf_conn = create_cf_connection(region) create_cf_stack(cf_conn, stack_name, cf_template_path, availability_zone) # create launcher instance ec2_conn = create_ec2_connection(region) launcher_instance = create_launcher_instance(ec2_conn, cf_conn, stack_name, launcher_ami, launcher_instance_type) # run bootstrap on launcher execute( run_director_bootstrap, director_conf_path=director_conf_path, region=region, cluster_ami=cluster_ami, num_workers=num_workers, stack_name=stack_name, hosts=[launcher_instance.ip_address], ) end_time = datetime.now() print "Cluster has started. Took {t} minutes.".format(t=(end_time - start_time).seconds / 60)
def teardown(region, stack_name): # terminate Hadoop cluster (prompts for confirmation) ec2_conn = create_ec2_connection(region) execute(run_director_terminate, hosts=[get_launcher_instance(ec2_conn, stack_name).ip_address]) # terminate launcher instance terminate_launcher_instance(ec2_conn, stack_name) # delete stack cf_conn = create_cf_connection(region) delete_stack(cf_conn, stack_name)
def run_director_bootstrap(director_conf_path, region, cluster_ami, num_workers, stack_name): # replace variables in conf template and copy to launcher cf_conn = create_cf_connection(region) params = { "accessKeyId": get_aws_access_key_id(), "secretAccessKey": get_aws_secret_access_key(), "region": region, "stack_name": stack_name, "owner": getuser(), "keyName": get_ec2_key_pair(), "subnetId": get_subnet_id(cf_conn, stack_name), "securityGroupsIds": get_security_group_id(cf_conn, stack_name), "image": cluster_ami, "num_workers": num_workers, } with open(director_conf_path, "r") as template_file: interpolated_body = template_file.read() % params director_conf = StringIO(interpolated_body) put(director_conf, "director.conf") # bootstrap the Hadoop cluster run("cloudera-director bootstrap director.conf")