示例#1
0
def launch_master(cluster, image_id, key_name, user_data_file_template=None,
    instance_type='m1.small', placement=None, user_packages=None,
    auto_shutdown=None, env_strings=[], client_cidrs=[], nfs_mount=None):
  if user_data_file_template == None:
    user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
  if cluster.check_running(MASTER, 0):
    return
  ebs_mappings=''
  storage = Storage(cluster)
  if storage.has_any_storage((MASTER,)):
    ebs_mappings = storage.get_mappings_string_for_role(MASTER)
  replacements = { "%ENV%": build_env_string(ENV_WHITELIST, env_strings, {
    "USER_PACKAGES": user_packages,
    "AUTO_SHUTDOWN": auto_shutdown,
    "EBS_MAPPINGS": ebs_mappings,
    "NFS_MOUNT": nfs_mount
  }) }
  reservation = cluster.launch_instances(MASTER, 1, image_id, key_name, user_data_file_template, replacements, instance_type, placement)
  print "Waiting for master to start (%s)" % str(reservation)
  cluster.wait_for_instances(reservation)
  print
  cluster.print_status((MASTER,))
  master = cluster.check_running(MASTER, 1)[0]
  _authorize_client_ports(cluster, master, client_cidrs)
  _create_client_hadoop_site_file(cluster, master)
示例#2
0
def launch_slaves(cluster,
                  number,
                  user_data_file_template=None,
                  user_packages=None,
                  auto_shutdown=None,
                  env_strings=[]):
    if user_data_file_template == None:
        user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
    instances = cluster.check_running(MASTER, 1)
    if not instances:
        return
    master = instances[0]
    ebs_mappings = ''
    storage = Storage(cluster)
    if storage.has_any_storage((SLAVE, )):
        ebs_mappings = storage.get_mappings_string_for_role(SLAVE)
    replacements = {
        "%ENV%":
        build_env_string(
            ENV_WHITELIST, env_strings, {
                "USER_PACKAGES": user_packages,
                "AUTO_SHUTDOWN": auto_shutdown,
                "EBS_MAPPINGS": ebs_mappings,
                "MASTER_HOST": master.public_dns_name
            })
    }
    reservation = cluster.launch_instances(SLAVE, number, master.image_id,
                                           master.key_name,
                                           user_data_file_template,
                                           replacements, master.instance_type,
                                           master.placement)
    print "Waiting for slaves to start"
    cluster.wait_for_instances(reservation)
    print
    cluster.print_status((SLAVE, ))
示例#3
0
def launch_slaves(cluster, number, user_data_file_template=None,
    user_packages=None, auto_shutdown=None, env_strings=[]):
  if user_data_file_template == None:
    user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
  instances = cluster.check_running(MASTER, 1)
  if not instances:
    return
  master = instances[0]
  ebs_mappings=''
  storage = Storage(cluster)
  if storage.has_any_storage((SLAVE,)):
    ebs_mappings = storage.get_mappings_string_for_role(SLAVE)
  replacements = { "%ENV%": build_env_string(ENV_WHITELIST, env_strings, {
    "USER_PACKAGES": user_packages,
    "AUTO_SHUTDOWN": auto_shutdown,
    "EBS_MAPPINGS": ebs_mappings,
    "MASTER_HOST": master.public_dns_name
  }) }
  while True:
    reservation = cluster.launch_instances(SLAVE, number, master.image_id, master.key_name, user_data_file_template,
      replacements, master.instance_type, master.placement)
    print "Waiting for slaves to start"
    try:
      cluster.wait_for_instances(reservation)
      print; cluster.print_status((SLAVE,))
      break
    except TimeoutException:
      print "Timeout while waiting for slaves to start. Cancelling reservation."
      cluster.terminate_reservation(reservation)
      time.sleep(60);
      print "Retrying."
示例#4
0
def launch_master(cluster,
                  image_id,
                  key_name,
                  user_data_file_template=None,
                  instance_type='m1.small',
                  placement=None,
                  user_packages=None,
                  auto_shutdown=None,
                  env_strings=[],
                  client_cidrs=[]):
    if user_data_file_template == None:
        user_data_file_template = DEFAULT_USER_DATA_FILE_TEMPLATE
    if cluster.check_running(MASTER, 0):
        return
    ebs_mappings = ''
    storage = Storage(cluster)
    if storage.has_any_storage((MASTER, )):
        ebs_mappings = storage.get_mappings_string_for_role(MASTER)
    replacements = {
        "%ENV%":
        build_env_string(
            ENV_WHITELIST, env_strings, {
                "USER_PACKAGES": user_packages,
                "AUTO_SHUTDOWN": auto_shutdown,
                "EBS_MAPPINGS": ebs_mappings
            })
    }
    reservation = cluster.launch_instances(MASTER, 1, image_id, key_name,
                                           user_data_file_template,
                                           replacements, instance_type,
                                           placement)
    print "Waiting for master to start (%s)" % str(reservation)
    cluster.wait_for_instances(reservation)
    print
    cluster.print_status((MASTER, ))
    master = cluster.check_running(MASTER, 1)[0]
    _authorize_client_ports(cluster, master, client_cidrs)
    _create_client_hadoop_site_file(cluster, master)