def launch_master(cluster, image_id, key_name, user_data_file_template=None,
instance_type='m1.small', placement=None, user_packages=None,
auto_shutdown=None, env_strings=[], client_cidrs=[], nfs_mount=None):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
if cluster.check_running(MASTER, 0):
return
ebs_mappings=''
storage = Storage(cluster)
if storage.has_any_storage((MASTER,)):
ebs_mappings = storage.get_mappings_string_for_role(MASTER)
replacements = { "%ENV%": build_env_string(ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings,
"NFS_MOUNT": nfs_mount
}) }
reservation = cluster.launch_instances(MASTER, 1, image_id, key_name, user_data_file_template, replacements, instance_type, placement)
print "Waiting for master to start (%s)" % str(reservation)
cluster.wait_for_instances(reservation)
print
cluster.print_status((MASTER,))
master = cluster.check_running(MASTER, 1)[0]
_authorize_client_ports(cluster, master, client_cidrs)
_create_client_hadoop_site_file(cluster, master)
def launch_slaves(cluster,
number,
user_data_file_template=None,
user_packages=None,
auto_shutdown=None,
env_strings=[]):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
instances = cluster.check_running(MASTER, 1)
if not instances:
return
master = instances[0]
ebs_mappings = ''
storage = Storage(cluster)
if storage.has_any_storage((SLAVE, )):
ebs_mappings = storage.get_mappings_string_for_role(SLAVE)
replacements = {
"%ENV%":
build_env_string(
ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings,
"MASTER_HOST": master.public_dns_name
})
}
reservation = cluster.launch_instances(SLAVE, number, master.image_id,
master.key_name,
user_data_file_template,
replacements, master.instance_type,
master.placement)
print "Waiting for slaves to start"
cluster.wait_for_instances(reservation)
print
cluster.print_status((SLAVE, ))
def launch_slaves(cluster, number, user_data_file_template=None,
user_packages=None, auto_shutdown=None, env_strings=[]):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
instances = cluster.check_running(MASTER, 1)
if not instances:
return
master = instances[0]
ebs_mappings=''
storage = Storage(cluster)
if storage.has_any_storage((SLAVE,)):
ebs_mappings = storage.get_mappings_string_for_role(SLAVE)
replacements = { "%ENV%": build_env_string(ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings,
"MASTER_HOST": master.public_dns_name
}) }
while True:
reservation = cluster.launch_instances(SLAVE, number, master.image_id, master.key_name, user_data_file_template,
replacements, master.instance_type, master.placement)
print "Waiting for slaves to start"
try:
cluster.wait_for_instances(reservation)
print; cluster.print_status((SLAVE,))
break
except TimeoutException:
print "Timeout while waiting for slaves to start. Cancelling reservation."
cluster.terminate_reservation(reservation)
time.sleep(60);
print "Retrying."
def launch_master(cluster,
image_id,
key_name,
user_data_file_template=None,
instance_type='m1.small',
placement=None,
user_packages=None,
auto_shutdown=None,
env_strings=[],
client_cidrs=[]):
if user_data_file_template == None:
user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
if cluster.check_running(MASTER, 0):
return
ebs_mappings = ''
storage = Storage(cluster)
if storage.has_any_storage((MASTER, )):
ebs_mappings = storage.get_mappings_string_for_role(MASTER)
replacements = {
"%ENV%":
build_env_string(
ENV_WHITELIST, env_strings, {
"USER_PACKAGES": user_packages,
"AUTO_SHUTDOWN": auto_shutdown,
"EBS_MAPPINGS": ebs_mappings
})
}
reservation = cluster.launch_instances(MASTER, 1, image_id, key_name,
user_data_file_template,
replacements, instance_type,
placement)
print "Waiting for master to start (%s)" % str(reservation)
cluster.wait_for_instances(reservation)
print
cluster.print_status((MASTER, ))
master = cluster.check_running(MASTER, 1)[0]
_authorize_client_ports(cluster, master, client_cidrs)
_create_client_hadoop_site_file(cluster, master)