def launch_slaves(cluster,
number,
user_data_file_template=None,
user_packages=None,
auto_shutdown=None,
env_strings=[]):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
instances = cluster.check_running(MASTER, 1)
if not instances:
return
master = instances[0]
ebs_mappings = ''
storage = Storage(cluster)
if storage.has_any_storage((SLAVE, )):
ebs_mappings = storage.get_mappings_string_for_role(SLAVE)
replacements = {
"%ENV%":
build_env_string(
ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings,
"MASTER_HOST": master.public_dns_name
})
}
reservation = cluster.launch_instances(SLAVE, number, master.image_id,
master.key_name,
user_data_file_template,
replacements, master.instance_type,
master.placement)
print "Waiting for slaves to start"
cluster.wait_for_instances(reservation)
print
cluster.print_status((SLAVE, ))
def attach_storage(cluster, roles):
storage = Storage(cluster)
if storage.has_any_storage(roles):
print "Waiting 10 seconds before attaching storage"
time.sleep(10)
for role in roles:
storage.attach(role,
cluster.get_instances_in_role(role, 'running'))
storage.print_status(roles)
def launch_master(cluster,
image_id,
key_name,
user_data_file_template=None,
instance_type='m1.small',
placement=None,
user_packages=None,
auto_shutdown=None,
env_strings=[],
client_cidrs=[]):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
if cluster.check_running(MASTER, 0):
return
ebs_mappings = ''
storage = Storage(cluster)
if storage.has_any_storage((MASTER, )):
ebs_mappings = storage.get_mappings_string_for_role(MASTER)
replacements = {
"%ENV%":
build_env_string(
ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings
})
}
reservation = cluster.launch_instances(MASTER, 1, image_id, key_name,
user_data_file_template,
replacements, instance_type,
placement)
print "Waiting for master to start (%s)" % str(reservation)
cluster.wait_for_instances(reservation)
print
cluster.print_status((MASTER, ))
master = cluster.check_running(MASTER, 1)[0]
_authorize_client_ports(cluster, master, client_cidrs)
_create_client_hadoop_site_file(cluster, master)
