def print_cluster_status(cls, options):
    """
    Gets cluster stats and prints it nicely.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    try:
      load_balancer_ip = LocalState.get_host_with_role(
        options.keyname, 'load_balancer')
      acc = AppControllerClient(
        load_balancer_ip, LocalState.get_secret_key(options.keyname))
      all_private_ips = acc.get_all_private_ips()
      cluster_stats = acc.get_cluster_stats()
    except (faultType, AppControllerException, BadConfigurationException):
      AppScaleLogger.warn("AppScale deployment is probably down")
      raise

    # Convert cluster stats to useful structures
    node_stats = {
      ip: next((n for n in cluster_stats if n["private_ip"] == ip), None)
      for ip in all_private_ips
    }
    apps_dict = next((n["apps"] for n in cluster_stats if n["apps"]), {})
    services = [ServiceInfo(key.split('_')[0], key.split('_')[1], app_info)
                for key, app_info in apps_dict.iteritems()]
    nodes = [NodeStats(ip, node) for ip, node in node_stats.iteritems() if node]
    invisible_nodes = [ip for ip, node in node_stats.iteritems() if not node]

    if options.verbose:
      AppScaleLogger.log("-"*76)
      cls._print_nodes_info(nodes, invisible_nodes)
      cls._print_roles_info(nodes)
    else:
      AppScaleLogger.log("-"*76)

    cls._print_cluster_summary(nodes, invisible_nodes, services)
    cls._print_services(services)
    cls._print_status_alerts(nodes)

    try:
      login_host = acc.get_property('login')['login']
    except KeyError:
      raise AppControllerException('login property not found')

    dashboard = next(
      (service for service in services
       if service.http == RemoteHelper.APP_DASHBOARD_PORT), None)
    if dashboard and dashboard.appservers >= 1:
      AppScaleLogger.success(
        "\nView more about your AppScale deployment at http://{}:{}/status"
        .format(login_host, RemoteHelper.APP_DASHBOARD_PORT)
      )
    else:
      AppScaleLogger.log(
        "\nAs soon as AppScale Dashboard is started you can visit it at "
        "http://{0}:{1}/status and see more about your deployment"
        .format(login_host, RemoteHelper.APP_DASHBOARD_PORT)
      )
  def create_user(cls, options, is_admin):
    """Create a new user with the parameters given.

        Args:
          options: A Namespace that has fields for each parameter that can be
            passed in via the command-line interface.
          is_admin: A flag to indicate if the user to be created is an admin user
        Raises:
          AppControllerException: If the AppController on the head node crashes.
            When this occurs, the message in the exception contains the reason why
            the AppController crashed.
        """
    secret = LocalState.get_secret_key(options.keyname)
    load_balancer_ip = LocalState.get_host_with_role(
      options.keyname, 'load_balancer')

    username, password = LocalState.get_credentials(is_admin)

    acc = AppControllerClient(load_balancer_ip, secret)

    RemoteHelper.create_user_accounts(
      username, password, load_balancer_ip, options.keyname)

    try:
      if is_admin:
        acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)
    except Exception as exception:
      AppScaleLogger.warn("Could not grant admin privileges to the user for the " +
        "following reason: {0}".format(str(exception)))
      sys.exit(1)
  def add_instances(cls, options):
    """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if 'master' in options.ips.keys():
      raise BadConfigurationException("Cannot add master nodes to an " + \
        "already running AppScale deployment.")

    # In virtualized cluster deployments, we need to make sure that the user
    # has already set up SSH keys.
    if LocalState.get_infrastructure_option(keyname=options.keyname,
                                            tag='infrastructure') == "xen":
      ips_to_check = []
      for ip_group in options.ips.values():
        ips_to_check.extend(ip_group)
      for ip in ips_to_check:
        # throws a ShellException if the SSH key doesn't work
        RemoteHelper.ssh(ip, options.keyname, "ls")

    # Finally, find an AppController and send it a message to add
    # the given nodes with the new roles.
    AppScaleLogger.log("Sending request to add instances")
    load_balancer_ip = LocalState.get_host_with_role(
      options.keyname, 'load_balancer')
    acc = AppControllerClient(load_balancer_ip, LocalState.get_secret_key(
      options.keyname))
    acc.start_roles_on_nodes(json.dumps(options.ips))

    # TODO(cgb): Should we wait for the new instances to come up and get
    # initialized?
    AppScaleLogger.success("Successfully sent request to add instances " + \
                           "to this AppScale deployment.")
Beispiel #4
0
 def test_set_deployment_id(self):
   host = 'boo'
   secret = 'baz'
   # The function should return whatever run_with_timeout_returns.
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return()
   acc = AppControllerClient(host, secret)
   acc.get_deployment_id()
Beispiel #5
0
 def test_get_deployment_id(self):
   # The function should return whatever run_with_timeout_returns.
   host = 'boo'
   secret = 'baz'
   deployment_id = 'foo'
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return(deployment_id)
   acc = AppControllerClient(host, secret)
   self.assertEqual(deployment_id, acc.get_deployment_id())
Beispiel #6
0
 def test_deployment_id_exists(self):
   # The function should return whatever run_with_timeout returns.
   host = 'boo'
   secret = 'baz'
   deployment_id_exists = True
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return(deployment_id_exists)
   acc = AppControllerClient(host, secret)
   self.assertEqual(deployment_id_exists, acc.deployment_id_exists())
  def set_property(cls, options):
    """Instructs AppScale to replace the value it uses for a particular
    AppController instance variable (property) with a new value.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    """
    shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow')
    acc = AppControllerClient(shadow_host, LocalState.get_secret_key(
      options.keyname))
    acc.set_property(options.property_name, options.property_value)
    AppScaleLogger.success('Successfully updated the given property.')
  def get_property(cls, options):
    """Queries AppScale for a list of system properties matching the provided
    regular expression, as well as the values associated with each matching
    property.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Returns:
      A dict mapping each property matching the given regex to its associated
      value.
    """
    shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow')
    acc = AppControllerClient(shadow_host, LocalState.get_secret_key(
      options.keyname))

    return acc.get_property(options.property)
    def test_appscale_in_one_node_virt_deployment_with_login_override(self):
        # let's say that appscale isn't already running
        self.local_state.should_receive(
            'ensure_appscale_isnt_running').and_return()
        self.local_state.should_receive('make_appscale_directory').and_return()
        self.local_state.should_receive('update_local_metadata').and_return()
        self.local_state.should_receive('get_local_nodes_info').and_return(
            json.loads(
                json.dumps([{
                    "public_ip": IP_1,
                    "private_ip": IP_1,
                    "roles": ["shadow"]
                }])))
        self.local_state.should_receive('get_secret_key').and_return("fookey")

        flexmock(RemoteHelper)
        RemoteHelper.should_receive('enable_root_ssh').and_return()
        RemoteHelper.should_receive('ensure_machine_is_compatible')\
            .and_return()
        RemoteHelper.should_receive('start_head_node')\
            .and_return((IP_1, 'i-ABCDEFG'))
        RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
        RemoteHelper.should_receive('copy_local_metadata').and_return()
        RemoteHelper.should_receive('create_user_accounts').and_return()
        RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
            .and_return()
        RemoteHelper.should_receive('copy_deployment_credentials')

        flexmock(AppControllerClient)
        AppControllerClient.should_receive('does_user_exist').and_return(True)
        AppControllerClient.should_receive('is_initialized').and_return(True)
        AppControllerClient.should_receive('set_admin_role').and_return('true')
        AppControllerClient.should_receive('get_property').\
          and_return({'login': IP_1})

        flexmock(AppScaleLogger)
        AppScaleLogger.should_receive('remote_log_tools_state').and_return()

        # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
        # machines to boot with those addresses (and that can mess up our tests).
        ips_layout = ONE_NODE_CLUSTER

        argv = [
            "--ips_layout",
            base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname,
            "--test", "--login_host", "www.booscale.com"
        ]

        options = ParseArgs(argv, self.function).args
        AppScaleTools.run_instances(options)
def get_roles(keyname):
    """
  Obtains roles for each ip from AppControllerClient.

  Args:
    keyname: A string representing an identifier from AppScaleFile.

  Returns:
    A dict in which each key is an ip and value is a role list.
  """
    load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer')
    acc = AppControllerClient(host=load_balancer_ip,
                              secret=LocalState.get_secret_key(keyname))
    cluster_stats = acc.get_cluster_stats()

    roles_data = {
        node["private_ip"]:
        (node["roles"] if len(node["roles"]) > 0 else ["?"])
        for node in cluster_stats
    }

    return roles_data
  def reset_password(cls, options):
    """Resets a user's password the currently running AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    secret = LocalState.get_secret_key(options.keyname)
    load_balancer_ip = LocalState.get_host_with_role(
      options.keyname, 'load_balancer')
    username, password = LocalState.get_credentials(is_admin=False)
    encrypted_password = LocalState.encrypt_password(username, password)

    acc = AppControllerClient(load_balancer_ip, secret)

    try:
      acc.reset_password(username, encrypted_password)
      AppScaleLogger.success("The password was successfully changed for the " \
        "given user.")
    except Exception as exception:
      AppScaleLogger.warn("Could not change the user's password for the " + \
        "following reason: {0}".format(str(exception)))
      sys.exit(1)
def get_roles(keyname):
  """
  Obtains roles for each ip from AppControllerClient.

  Args:
    keyname: A string representing an identifier from AppScaleFile.

  Returns:
    A dict in which each key is an ip and value is a role list.
  """
  load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer')
  acc = AppControllerClient(
    host=load_balancer_ip,
    secret=LocalState.get_secret_key(keyname)
  )
  cluster_stats = acc.get_cluster_stats()

  roles_data = {
    node["private_ip"]: (node["roles"] if len(node["roles"]) > 0 else ["?"])
    for node in cluster_stats
  }

  return roles_data
  def test_appscale_in_one_node_virt_deployment_with_login_override(self):
    # let's say that appscale isn't already running
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('update_local_metadata').and_return()
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : IP_1,
        "private_ip" : IP_1,
        "roles" : ["shadow"]
      }])))
    self.local_state.should_receive('get_secret_key').and_return("fookey")

    flexmock(RemoteHelper)
    RemoteHelper.should_receive('enable_root_ssh').and_return()
    RemoteHelper.should_receive('ensure_machine_is_compatible')\
        .and_return()
    RemoteHelper.should_receive('start_head_node')\
        .and_return((IP_1, 'i-ABCDEFG'))
    RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
    RemoteHelper.should_receive('copy_local_metadata').and_return()
    RemoteHelper.should_receive('create_user_accounts').and_return()
    RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
        .and_return()
    RemoteHelper.should_receive('copy_deployment_credentials')

    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)
    AppControllerClient.should_receive('is_initialized').and_return(True)
    AppControllerClient.should_receive('set_admin_role').and_return('true')
    AppControllerClient.should_receive('get_property').\
      and_return({'login': IP_1})

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = ONE_NODE_CLUSTER

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test",
      "--login_host", "www.booscale.com"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
  def relocate_app(cls, options):
    """Instructs AppScale to move the named application to a different port.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Raises:
      AppScaleException: If the named application isn't running in this AppScale
        cloud, if the destination port is in use by a different application, or
        if the AppController rejects the request to relocate the application (in
        which case it includes the reason why the rejection occurred).
    """
    load_balancer_ip = LocalState.get_host_with_role(
      options.keyname, 'load_balancer')
    acc = AppControllerClient(
      load_balancer_ip, LocalState.get_secret_key(options.keyname))

    version_key = '_'.join([options.appname, DEFAULT_SERVICE, DEFAULT_VERSION])
    app_info_map = acc.get_app_info_map()
    if version_key not in app_info_map:
      raise AppScaleException("The given application, {0}, is not currently " \
        "running in this AppScale cloud, so we can't move it to a different " \
        "port.".format(options.appname))

    try:
      login_host = acc.get_property('login')['login']
    except KeyError:
      raise AppControllerException('login property not found')

    acc.relocate_version(version_key, options.http_port, options.https_port)
    AppScaleLogger.success(
      'Successfully issued request to move {0} to ports {1} and {2}'.format(
        options.appname, options.http_port, options.https_port))
    RemoteHelper.sleep_until_port_is_open(login_host, options.http_port)
    AppScaleLogger.success(
      'Your app serves unencrypted traffic at: http://{0}:{1}'.format(
        login_host, options.http_port))
    AppScaleLogger.success(
      'Your app serves encrypted traffic at: https://{0}:{1}'.format(
        login_host, options.https_port))
Beispiel #15
0
  def test_appscale_in_one_node_virt_deployment(self):
    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null root@public1 ",
                False, 5,
                stdin="cp /root/appscale/AppController/scripts/appcontroller "
                      "/etc/init.d/")

    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null [email protected] ",
                False, 5, stdin="chmod +x /etc/init.d/appcontroller")
    
    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null root@public1 ",
                False, 5,
                stdin="cp /root/appscale/AppController/scripts/appcontroller "
                      "/etc/init.d/")

    # let's say that appscale isn't already running
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()

    rh = flexmock(RemoteHelper)
    rh.should_receive('copy_deployment_credentials').and_return()

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').\
      with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').\
      with_args('logs.appscale.com').and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').\
      with_args(secret_key_location, 'r').and_return(fake_secret)
    self.builtins.should_receive('open').\
      with_args(secret_key_location, 'w').and_return(fake_secret)

    # Don't write local metadata files.
    flexmock(LocalState).should_receive('update_local_metadata')

    # mock out copying over the keys
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.key'),False,5)

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^openssl'),False,stdin=None)\
      .and_return()

    # mock out removing the old json file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
      .and_return()

    # assume that we started monit fine
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
      .and_return()

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
      .and_return()

    self.local_state.should_receive('shell').\
      with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
                '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
                '-o UserKnownHostsFile=/dev/null [email protected] ',
                False, 5,
                stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.setup_socket_mocks('1.2.3.4')
    self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip": "1.2.3.4",
        "private_ip": "1.2.3.4",
        "jobs": ["shadow", "login"]
      }])))

    # Assume the locations files were copied successfully.
    locations_file = '{}/locations-bookey.yaml'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
      .and_return()

    locations_json = '{}/locations-bookey.json'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
      .and_return()

    user_locations = '/root/.appscale/locations-bookey.json'
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
      .and_return()

    # Assume the secret key was copied successfully.
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.secret'), False, 5)\
      .and_return()

    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine:  1.2.3.4
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
  def test_appscale_in_one_node_virt_deployment(self):
    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null root@public1 ",
                False, 5,
                stdin="cp /root/appscale/AppController/scripts/appcontroller "
                      "/etc/init.d/")

    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null [email protected] ",
                False, 5, stdin="chmod +x /etc/init.d/appcontroller")
    
    self.local_state.should_receive('shell').\
      with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
                "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
                "-o UserKnownHostsFile=/dev/null root@public1 ",
                False, 5,
                stdin="cp /root/appscale/AppController/scripts/appcontroller "
                      "/etc/init.d/")

    # let's say that appscale isn't already running
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()

    rh = flexmock(RemoteHelper)
    rh.should_receive('copy_deployment_credentials').and_return()

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').\
      with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').\
      with_args('logs.appscale.com').and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').\
      with_args(secret_key_location, 'r').and_return(fake_secret)
    self.builtins.should_receive('open').\
      with_args(secret_key_location, 'w').and_return(fake_secret)

    # Don't write local metadata files.
    flexmock(LocalState).should_receive('update_local_metadata')

    # mock out copying over the keys
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.key'),False,5)

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^openssl'),False,stdin=None)\
      .and_return()

    # mock out removing the old json file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
      .and_return()

    # assume that we started monit fine
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
      .and_return()

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
      .and_return()

    self.local_state.should_receive('shell').\
      with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
                '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
                '-o UserKnownHostsFile=/dev/null [email protected] ',
                False, 5,
                stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.setup_socket_mocks('1.2.3.4')
    self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip": "1.2.3.4",
        "private_ip": "1.2.3.4",
        "jobs": ["shadow", "login"]
      }])))

    # Assume the locations files were copied successfully.
    locations_file = '{}/locations-bookey.yaml'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
      .and_return()

    locations_json = '{}/locations-bookey.json'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
      .and_return()

    user_locations = '/root/.appscale/locations-bookey.json'
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
      .and_return()

    # Assume the secret key was copied successfully.
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.secret'), False, 5)\
      .and_return()

    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine:  1.2.3.4
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
  def run_instances(cls, options):
    """Starts a new AppScale deployment with the parameters given.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        When this occurs, the message in the exception contains the reason why
        the AppController crashed.
      BadConfigurationException: If the user passes in options that are not
        sufficient to start an AppScale deployment (e.g., running on EC2 but
        not specifying the AMI to use), or if the user provides us
        contradictory options (e.g., running on EC2 but not specifying EC2
        credentials).
    """
    LocalState.make_appscale_directory()
    LocalState.ensure_appscale_isnt_running(options.keyname, options.force)
    node_layout = NodeLayout(options)

    if options.infrastructure:
      if (not options.test and not options.force and
          not (options.disks or node_layout.are_disks_used())):
        LocalState.ensure_user_wants_to_run_without_disks()

    reduced_version = '.'.join(x for x in APPSCALE_VERSION.split('.')[:2])
    AppScaleLogger.log("Starting AppScale " + reduced_version)

    my_id = str(uuid.uuid4())
    AppScaleLogger.remote_log_tools_state(options, my_id, "started",
      APPSCALE_VERSION)

    head_node = node_layout.head_node()
    # Start VMs in cloud via cloud agent.
    if options.infrastructure:
      node_layout = RemoteHelper.start_all_nodes(options, node_layout)

      # Enables root logins and SSH access on the head node.
      RemoteHelper.enable_root_ssh(options, head_node.public_ip)
    AppScaleLogger.verbose("Node Layout: {}".format(node_layout.to_list()))

    # Ensure all nodes are compatible.
    RemoteHelper.ensure_machine_is_compatible(
      head_node.public_ip, options.keyname)

    # Use rsync to move custom code into the deployment.
    if options.rsync_source:
      AppScaleLogger.log("Copying over local copy of AppScale from {0}".
        format(options.rsync_source))
      RemoteHelper.rsync_files(head_node.public_ip, options.keyname,
                               options.rsync_source)

    # Start services on head node.
    RemoteHelper.start_head_node(options, my_id, node_layout)

    # Write deployment metadata to disk (facilitates SSH operations, etc.)
    db_master = node_layout.db_master().private_ip
    head_node = node_layout.head_node().public_ip
    LocalState.update_local_metadata(options, db_master, head_node)

    # Copy the locations.json to the head node
    RemoteHelper.copy_local_metadata(node_layout.head_node().public_ip,
                                     options.keyname)

    # Wait for services on head node to start.
    secret_key = LocalState.get_secret_key(options.keyname)
    acc = AppControllerClient(head_node, secret_key)
    try:
      while not acc.is_initialized():
        AppScaleLogger.log('Waiting for head node to initialize...')
        # This can take some time in particular the first time around, since
        # we will have to initialize the database.
        time.sleep(cls.SLEEP_TIME*3)
    except socket.error as socket_error:
      AppScaleLogger.warn('Unable to initialize AppController: {}'.
                          format(socket_error.message))
      message = RemoteHelper.collect_appcontroller_crashlog(
        head_node, options.keyname)
      raise AppControllerException(message)

    # Set up admin account.
    try:
      # We don't need to have any exception information here: we do expect
      # some anyway while the UserAppServer is coming up.
      acc.does_user_exist("non-existent-user", True)
    except Exception:
      AppScaleLogger.log('UserAppServer not ready yet. Retrying ...')
      time.sleep(cls.SLEEP_TIME)

    if options.admin_user and options.admin_pass:
      AppScaleLogger.log("Using the provided admin username/password")
      username, password = options.admin_user, options.admin_pass
    elif options.test:
      AppScaleLogger.log("Using default admin username/password")
      username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
    else:
      username, password = LocalState.get_credentials()

    RemoteHelper.create_user_accounts(username, password, head_node,
                                      options.keyname)
    acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)

    # Wait for machines to finish loading and AppScale Dashboard to be deployed.
    RemoteHelper.wait_for_machines_to_finish_loading(head_node, options.keyname)

    try:
      login_host = acc.get_property('login')['login']
    except KeyError:
      raise AppControllerException('login property not found')

    RemoteHelper.sleep_until_port_is_open(
      login_host, RemoteHelper.APP_DASHBOARD_PORT)

    AppScaleLogger.success("AppScale successfully started!")
    AppScaleLogger.success(
      'View status information about your AppScale deployment at '
      'http://{}:{}'.format(login_host, RemoteHelper.APP_DASHBOARD_PORT))
    AppScaleLogger.remote_log_tools_state(options, my_id,
      "finished", APPSCALE_VERSION)
  def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
    # let's say that appscale isn't already running
    local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
      os.sep + self.keyname + ".key"
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('get_key_path_from_name').and_return(
      local_appscale_path)

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    self.setup_ec2_mocks()

    # also mock out acquiring a spot instance
    self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
      'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
      instance_type='m3.medium', count=1, placement='my-zone-1b')

    # Don't write local metadata files.
    flexmock(LocalState).should_receive('update_local_metadata')

    # assume that root login is not enabled
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)

    # assume that we can enable root login
    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo touch /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin='mktemp').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sed -n '
        '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
    ).and_return()

    # and assume that we can copy over our ssh keys fine
    self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
      False, 5).and_return()
    self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
      .format(self.keyname)), False, 5).and_return()

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell').with_args(re.compile('openssl'),
      False, stdin=None)

    # assume that we started monit fine
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin=re.compile('monit'))

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('controller-17443.cfg'))

    self.setup_socket_mocks('public1')
    self.setup_appcontroller_mocks('public1', 'private1')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "public1",
        "private_ip" : "private1",
        "jobs" : ["shadow", "login"]
      }])))

    # copying over the locations json file should be fine
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))

    # same for the secret key
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    argv = [
      "--min", "1",
      "--max", "1",
      "--infrastructure", "ec2",
      "--instance_type", "m3.medium",
      "--machine", "ami-ABCDEFG",
      "--use_spot_instances",
      "--max_spot_price", "1.23",
      "--keyname", self.keyname,
      "--group", self.group,
      "--test",
      "--zone", "my-zone-1b"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
    def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
        # let's say that appscale isn't already running
        local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
          os.sep + self.keyname + ".key"
        self.local_state.should_receive(
            'ensure_appscale_isnt_running').and_return()
        self.local_state.should_receive('make_appscale_directory').and_return()
        self.local_state.should_receive('get_key_path_from_name').and_return(
            local_appscale_path)

        # mock out talking to logs.appscale.com
        fake_connection = flexmock(name='fake_connection')
        fake_connection.should_receive('request').with_args(
            'POST', '/upload', str, AppScaleLogger.HEADERS).and_return()

        flexmock(httplib)
        httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
          .and_return(fake_connection)

        # mock out generating the secret key
        flexmock(uuid)
        uuid.should_receive('uuid4').and_return('the secret')

        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        fake_secret.should_receive('write').and_return()
        self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)
        self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
          .and_return(fake_secret)

        self.setup_ec2_mocks()

        # also mock out acquiring a spot instance
        self.fake_ec2.should_receive('request_spot_instances').with_args(
            '1.23',
            'ami-ABCDEFG',
            key_name=self.keyname,
            network_interfaces=None,
            security_groups=[self.group],
            instance_type='m3.medium',
            count=1,
            placement='my-zone-1b')
        # Don't write local metadata files.
        flexmock(LocalState).should_receive('update_local_metadata')

        # assume that root login is not enabled
        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'), None, 5, stdin='ls').and_return(
                'Please login as the user "ubuntu" rather than the user "root"'
            )

        # assume that we can enable root login
        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'),
            None,
            5,
            stdin='sudo touch /root/.ssh/authorized_keys').and_return()

        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'),
            None,
            5,
            stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()

        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'), None, 5, stdin='mktemp').and_return()

        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'),
            None,
            5,
            stdin=re.compile(
                'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
            )).and_return()

        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'),
            None,
            5,
            stdin=re.compile(
                'sudo sed -n '
                '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ')
        ).and_return()

        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'), None, 5,
            stdin=re.compile('rm -f ')).and_return()
        self.local_state.should_receive('shell').with_args(
            re.compile('ssh'), None, 5,
            stdin=re.compile('rm -rf ')).and_return()

        # and assume that we can copy over our ssh keys fine
        self.local_state.should_receive('shell').with_args(
            re.compile('scp .*[r|d]sa'), None, 5).and_return()
        self.local_state.should_receive('shell').with_args(
            re.compile('scp .*{0}'.format(self.keyname)), None,
            5).and_return()

        self.setup_appscale_compatibility_mocks()

        # mock out generating the private key
        self.local_state.should_receive('shell').with_args(
            re.compile('openssl'), None, stdin=None)

        self.local_state.should_receive('shell').with_args(
            re.compile('^ssh'),
            None,
            5,
            stdin='systemctl start appscale-controller')

        self.setup_socket_mocks('public1')
        self.setup_appcontroller_mocks('public1', 'private1')

        # mock out reading the locations.json file, and slip in our own json
        self.local_state.should_receive('get_local_nodes_info').and_return(
            json.loads(
                json.dumps([{
                    "public_ip": "public1",
                    "private_ip": "private1",
                    "roles": ["shadow"]
                }])))

        # copying over the locations json file should be fine
        self.local_state.should_receive('shell').with_args(
            re.compile('scp'),
            None,
            5,
            stdin=re.compile('locations-{0}'.format(self.keyname)))

        # same for the secret key
        self.local_state.should_receive('shell').with_args(
            re.compile('scp'),
            None,
            5,
            stdin=re.compile('{0}.secret'.format(self.keyname)))

        flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
        flexmock(AppControllerClient)
        AppControllerClient.should_receive('does_user_exist').and_return(True)
        AppControllerClient.should_receive('get_property').\
          and_return({'login': '******'})

        # Let's mock the call to describe_instances when checking for old
        # instances to re-use, and then to start the headnode.
        pending_instance = flexmock(name='pending_instance',
                                    state='pending',
                                    key_name=self.keyname,
                                    id='i-ABCDEFG')
        pending_reservation = flexmock(name='pending_reservation',
                                       instances=[pending_instance])

        no_instances = flexmock(name='no_instances', instances=[])
        running_instance = flexmock(name='running_instance',
                                    state='running',
                                    key_name=self.keyname,
                                    id='i-ABCDEFG',
                                    ip_address='public1',
                                    private_ip_address='private1')
        running_reservation = flexmock(name='running_reservation',
                                       instances=[running_instance])

        self.fake_ec2.should_receive('get_all_instances') \
          .and_return(no_instances) \
          .and_return(pending_reservation) \
          .and_return(running_reservation)

        argv = [
            "--min", "1", "--max", "1", "--infrastructure", "ec2",
            "--instance_type", "m3.medium", "--machine", "ami-ABCDEFG",
            "--use_spot_instances", "--max_spot_price", "1.23", "--keyname",
            self.keyname, "--group", self.group, "--test", "--zone",
            "my-zone-1b", "--EC2_ACCESS_KEY", "baz", "--EC2_SECRET_KEY", "baz"
        ]

        options = ParseArgs(argv, self.function).args
        AppScaleTools.run_instances(options)
  def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self):
    # let's say that appscale isn't already running
    local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
      os.sep + self.keyname + ".key"
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('get_key_path_from_name').and_return(
      local_appscale_path)

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    self.setup_ec2_mocks()

    # slip in some fake spot instance info
    fake_entry = flexmock(name='fake_entry', price=1)
    self.fake_ec2.should_receive('get_spot_price_history').with_args(
      start_time=str, end_time=str,
      product_description='Linux/UNIX', instance_type='m3.medium',
      availability_zone='my-zone-1b').and_return([fake_entry])

    # also mock out acquiring a spot instance
    self.fake_ec2.should_receive('request_spot_instances').with_args('1.1',
      'ami-ABCDEFG', key_name=self.keyname, security_groups=[self.group],
      instance_type='m3.medium', count=1, placement='my-zone-1b')

    # Don't write local metadata files.
    flexmock(LocalState).should_receive('update_local_metadata')

    # assume that root login is not enabled
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin='ls').and_return(
      'Please login as the user "ubuntu" rather than the user "root"')

    # assume that we can enable root login
    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo touch /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin='mktemp').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sed -n '
        '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
    ).and_return()

    # and assume that we can copy over our ssh keys fine
    self.local_state.should_receive('shell').\
      with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return()
    self.local_state.should_receive('shell').\
      with_args(re.compile('scp .*{0}'.format(self.keyname)), False, 5).\
      and_return()

    self.local_state.should_receive('shell').\
      with_args('ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o '
                'NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
                '-o UserKnownHostsFile=/dev/null root@public1 ',
                False, 5,
                stdin='cp /root/appscale/AppController/scripts/appcontroller '
                      '/etc/init.d/').\
      and_return()

    self.local_state.should_receive('shell').\
      with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
                '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
                '-o UserKnownHostsFile=/dev/null root@elastic-ip ',
                False, 5,
                stdin='cp /root/appscale/AppController/scripts/appcontroller '
                      '/etc/init.d/').\
      and_return()

    self.local_state.should_receive('shell').\
      with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
                '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
                '-o UserKnownHostsFile=/dev/null root@elastic-ip ',
                False, 5, stdin='chmod +x /etc/init.d/appcontroller').\
      and_return()

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell').with_args(re.compile('openssl'),
      False, stdin=None)

    # assume that we started monit fine
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin=re.compile('monit'))

    self.local_state.should_receive('shell').with_args(
      re.compile('^ssh'), False, 5, stdin='service appscale-controller start')

    self.setup_socket_mocks('elastic-ip')
    self.setup_appcontroller_mocks('elastic-ip', 'private1')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "elastic-ip",
        "private_ip" : "private1",
        "jobs": ["shadow", "login"]
      }])))

    # copying over the locations yaml and json files should be fine
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))

    # same for the secret key
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))

    flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    # Let's mock the call to describe_instances when checking for old
    # instances to re-use, and then to start the headnode.
    pending_instance = flexmock(name='pending_instance', state='pending',
      key_name=self.keyname, id='i-ABCDEFG')
    pending_reservation = flexmock(name='pending_reservation',
      instances=[pending_instance])

    no_instances = flexmock(name='no_instances', instances=[])
    running_instance = flexmock(name='running_instance', state='running',
      key_name=self.keyname, id='i-ABCDEFG', ip_address='public1',
      private_ip_address='private1')
    running_reservation = flexmock(name='running_reservation',
      instances=[running_instance])

    self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
      .and_return(no_instances) \
      .and_return(no_instances).and_return(pending_reservation) \
      .and_return(running_reservation)

    argv = [
      "--min", "1",
      "--max", "1",
      "--infrastructure", "ec2",
      "--machine", "ami-ABCDEFG",
      "--instance_type", "m3.medium",
      "--use_spot_instances",
      "--keyname", self.keyname,
      "--group", self.group,
      "--test",
      "--zone", "my-zone-1b",
      "--static_ip", "elastic-ip",
      "--EC2_ACCESS_KEY", "baz",
      "--EC2_SECRET_KEY", "baz"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    location = os.path.abspath(options.location)
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {}, already exists.".format(location))

    load_balancer_ip = LocalState.get_host_with_role(
      options.keyname, 'load_balancer')
    secret = LocalState.get_secret_key(options.keyname)
    acc = AppControllerClient(load_balancer_ip, secret)

    try:
      all_ips = acc.get_all_public_ips()
    except socket.error:  # Occurs when the AppController has failed.
      AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
        "machines in this AppScale deployment. Using our locally cached " + \
        "info instead.")
      all_ips = LocalState.get_all_public_ips(options.keyname)

    # Get information about roles and public IPs
    # for creating navigation symlinks in gathered logs
    try:
      nodes_info = acc.get_role_info()
    except socket.error:  # Occurs when the AppController has failed.
      AppScaleLogger.warn("Couldn't get an up-to-date nodes info. "
                          "Using our locally cached info instead.")
      nodes_info = LocalState.get_local_nodes_info(options.keyname)
    nodes_dict = {node['public_ip']: node for node in nodes_info}

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(location)

    # make dir for private IP navigation links
    private_ips_dir = os.path.join(location, 'symlinks', 'private-ips')
    utils.mkdir(private_ips_dir)

    # The log paths that we collect logs from.
    log_paths = [
      {'remote': '/opt/cassandra/cassandra/logs/*', 'local': 'cassandra'},
      {'remote': '/var/log/appscale'},
      {'remote': '/var/log/haproxy.log*'},
      {'remote': '/var/log/kern.log*'},
      {'remote': '/var/log/nginx'},
      {'remote': '/var/log/rabbitmq/*', 'local': 'rabbitmq'},
      {'remote': '/var/log/syslog*'},
      {'remote': '/var/log/zookeeper'}
    ]

    failures = False
    for public_ip in all_ips:
      # Get the logs from each node, and store them in our local directory
      local_dir = os.path.join(location, public_ip)
      utils.mkdir(local_dir)
      local_link = os.path.join('..', '..', public_ip)

      # Create symlinks for easier navigation in gathered logs
      node_info = nodes_dict.get(public_ip)
      if node_info:
        private_ip_dir = os.path.join(private_ips_dir, node_info["private_ip"])
        os.symlink(local_link, private_ip_dir)
        for role in node_info['roles']:
          role_dir = os.path.join(location, 'symlinks', role)
          utils.mkdir(role_dir)
          os.symlink(local_link, os.path.join(role_dir, public_ip))

      for log_path in log_paths:
        sub_dir = local_dir

        if 'local' in log_path:
          sub_dir = os.path.join(local_dir, log_path['local'])
          utils.mkdir(sub_dir)

        try:
          RemoteHelper.scp_remote_to_local(
            public_ip, options.keyname, log_path['remote'], sub_dir
          )
        except ShellException as shell_exception:
          failures = True
          AppScaleLogger.warn('Unable to collect logs from {} for host {}'.
                              format(log_path['remote'], public_ip))
          AppScaleLogger.verbose(
            'Encountered exception: {}'.format(str(shell_exception)))

    if failures:
      AppScaleLogger.log("Done copying to {}. There were failures while "
                         "collecting AppScale logs.".format(location))
    else:
      AppScaleLogger.success("Successfully collected all AppScale logs into "
                             "{}".format(location))
Beispiel #22
0
  def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
    # let's say that appscale isn't already running
    local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
      os.sep + self.keyname + ".key"
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('get_key_path_from_name').and_return(
      local_appscale_path)

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    self.setup_ec2_mocks()

    # also mock out acquiring a spot instance
    self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
      'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
      instance_type='m3.medium', count=1, placement='my-zone-1b')

    # Don't write local metadata files.
    flexmock(LocalState).should_receive('update_local_metadata')

    # assume that root login is not enabled
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)

    # assume that we can enable root login
    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo touch /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin='mktemp').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sed -n '
        '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
    ).and_return()

    # and assume that we can copy over our ssh keys fine
    self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
      False, 5).and_return()
    self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
      .format(self.keyname)), False, 5).and_return()

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell').with_args(re.compile('openssl'),
      False, stdin=None)

    # assume that we started monit fine
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin=re.compile('monit'))

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('controller-17443.cfg'))

    self.setup_socket_mocks('public1')
    self.setup_appcontroller_mocks('public1', 'private1')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "public1",
        "private_ip" : "private1",
        "jobs" : ["shadow", "login"]
      }])))

    # copying over the locations json file should be fine
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))

    # same for the secret key
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    # Let's mock the call to describe_instances when checking for old
    # instances to re-use, and then to start the headnode.
    pending_instance = flexmock(name='pending_instance', state='pending',
                                key_name=self.keyname, id='i-ABCDEFG')
    pending_reservation = flexmock(name='pending_reservation',
                                   instances=[pending_instance])

    no_instances = flexmock(name='no_instances', instances=[])
    running_instance = flexmock(name='running_instance', state='running',
                                key_name=self.keyname, id='i-ABCDEFG',
                                ip_address='public1',
                                private_ip_address='private1')
    running_reservation = flexmock(name='running_reservation',
                                   instances=[running_instance])

    self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
      .and_return(no_instances) \
      .and_return(no_instances).and_return(pending_reservation) \
      .and_return(running_reservation)

    argv = [
      "--min", "1",
      "--max", "1",
      "--infrastructure", "ec2",
      "--instance_type", "m3.medium",
      "--machine", "ami-ABCDEFG",
      "--use_spot_instances",
      "--max_spot_price", "1.23",
      "--keyname", self.keyname,
      "--group", self.group,
      "--test",
      "--zone", "my-zone-1b"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)