Пример #1
0
  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(options.location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {0}, already exists.".format(options.location))

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(options.location)

    for ip in acc.get_all_public_ips():
      # Get the logs from each node, and store them in our local directory
      local_dir = "{0}/{1}".format(options.location, ip)
      os.mkdir(local_dir)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
        local_dir, options.verbose)
    AppScaleLogger.success("Successfully copied logs to {0}".format(
      options.location))
Пример #2
0
    def remove_app(cls, options):
        """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if not options.confirm:
            response = raw_input("Are you sure you want to remove this " + \
              "application? (Y/N) ")
            if response not in ['y', 'yes', 'Y', 'YES']:
                raise AppScaleException("Cancelled application removal.")

        login_host = LocalState.get_login_host(options.keyname)
        acc = AppControllerClient(login_host,
                                  LocalState.get_secret_key(options.keyname))
        userappclient = UserAppClient(
            login_host, LocalState.get_secret_key(options.keyname))
        if not userappclient.does_app_exist(options.appname):
            raise AppScaleException(
                "The given application is not currently running.")

        acc.stop_app(options.appname)
        AppScaleLogger.log("Please wait for your app to shut down.")
        while True:
            if acc.is_app_running(options.appname):
                time.sleep(cls.SLEEP_TIME)
            else:
                break
        AppScaleLogger.success("Done shutting down {0}".format(
            options.appname))
Пример #3
0
  def ensure_appscale_isnt_running(cls, keyname, force):
    """Checks the secret key file to see if AppScale is running, and
    aborts if it is.

    Args:
      keyname: The keypair name that is used to identify AppScale deployments.
      force: A bool that is used to run AppScale even if the secret key file
        is present.
    Raises:
      BadConfigurationException: If AppScale is already running.
    """
    if force:
      return

    if os.path.exists(cls.get_secret_key_location(keyname)):
      try:
        login_host = cls.get_login_host(keyname)
        secret_key = cls.get_secret_key(keyname)
      except (IOError, AppScaleException, BadConfigurationException):
        # If we don't have the locations files, we are not running.
        return

      acc = AppControllerClient(login_host, secret_key)
      try:
        acc.get_all_public_ips()
      except AppControllerException:
        # AC is not running, so we assume appscale is not up and running.
        AppScaleLogger.log("AppController not running on login node.")
      else:
        raise BadConfigurationException("AppScale is already running. Terminate" +
          " it, set 'force: True' in your AppScalefile, or use the --force flag" +
          " to run anyways.")
Пример #4
0
  def relocate_app(cls, options):
    """Instructs AppScale to move the named application to a different port.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Raises:
      AppScaleException: If the named application isn't running in this AppScale
        cloud, if the destination port is in use by a different application, or
        if the AppController rejects the request to relocate the application (in
        which case it includes the reason why the rejection occurred).
    """
    login_host = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_host, LocalState.get_secret_key(
      options.keyname))

    app_info_map = acc.get_app_info_map()
    if options.appname not in app_info_map.keys():
      raise AppScaleException("The given application, {0}, is not currently " \
        "running in this AppScale cloud, so we can't move it to a different " \
        "port.".format(options.appname))

    relocate_result = acc.relocate_app(options.appname, options.http_port,
      options.https_port)
    if relocate_result == "OK":
      AppScaleLogger.success("Successfully issued request to move {0} to " \
        "ports {1} and {2}.".format(options.appname, options.http_port,
        options.https_port))
      AppScaleLogger.success("Your app serves unencrypted traffic at: " +
        "http://{0}:{1}".format(login_host, options.http_port))
      AppScaleLogger.success("Your app serves encrypted traffic at: " +
        "https://{0}:{1}".format(login_host, options.https_port))
    else:
      raise AppScaleException(relocate_result)
Пример #5
0
    def add_instances(cls, options):
        """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if 'master' in options.ips.keys():
            raise BadConfigurationException("Cannot add master nodes to an " + \
              "already running AppScale deployment.")

        # Skip checking for -n (replication) because we don't allow the user
        # to specify it here (only allowed in run-instances).
        additional_nodes_layout = NodeLayout(options)

        # In virtualized cluster deployments, we need to make sure that the user
        # has already set up SSH keys.
        if LocalState.get_from_yaml(options.keyname,
                                    'infrastructure') == "xen":
            for ip in options.ips.values():
                # throws a ShellException if the SSH key doesn't work
                RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)

        # Finally, find an AppController and send it a message to add
        # the given nodes with the new roles.
        AppScaleLogger.log("Sending request to add instances")
        login_ip = LocalState.get_login_host(options.keyname)
        acc = AppControllerClient(login_ip,
                                  LocalState.get_secret_key(options.keyname))
        acc.start_roles_on_nodes(json.dumps(options.ips))

        # TODO(cgb): Should we wait for the new instances to come up and get
        # initialized?
        AppScaleLogger.success("Successfully sent request to add instances " + \
          "to this AppScale deployment.")
Пример #6
0
  def ensure_appscale_isnt_running(cls, keyname, force):
    """Checks the secret key file to see if AppScale is running, and
    aborts if it is.

    Args:
      keyname: The keypair name that is used to identify AppScale deployments.
      force: A bool that is used to run AppScale even if the secret key file
        is present.
    Raises:
      BadConfigurationException: If AppScale is already running.
    """
    if force:
      return

    if os.path.exists(cls.get_secret_key_location(keyname)):
      try:
        login_host = cls.get_login_host(keyname)
        secret_key = cls.get_secret_key(keyname)
      except (IOError, AppScaleException, BadConfigurationException):
        # If we don't have the locations files, we are not running.
        return

      acc = AppControllerClient(login_host, secret_key)
      try:
        acc.get_all_public_ips()
      except AppControllerException:
        # AC is not running, so we assume appscale is not up and running.
        AppScaleLogger.log("AppController not running on login node.")
      else:
        raise BadConfigurationException("AppScale is already running. Terminate" +
          " it, set 'force: True' in your AppScalefile, or use the --force flag" +
          " to run anyways.")
Пример #7
0
  def remove_app(cls, options):
    """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if not options.confirm:
      response = raw_input("Are you sure you want to remove this " + \
        "application? (Y/N) ")
      if response not in ['y', 'yes', 'Y', 'YES']:
        raise AppScaleException("Cancelled application removal.")

    login_host = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_host, LocalState.get_secret_key(
      options.keyname))
    userappserver_host = acc.get_uaserver_host(options.verbose)
    userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
      options.keyname))
    if not userappclient.does_app_exist(options.appname):
      raise AppScaleException("The given application is not currently running.")

    acc.stop_app(options.appname)
    AppScaleLogger.log("Please wait for your app to shut down.")
    while True:
      if acc.is_app_running(options.appname):
        time.sleep(cls.SLEEP_TIME)
      else:
        break
    AppScaleLogger.success("Done shutting down {0}".format(options.appname))
Пример #8
0
  def add_instances(cls, options):
    """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if 'master' in options.ips.keys():
      raise BadConfigurationException("Cannot add master nodes to an " + \
        "already running AppScale deployment.")

    # Skip checking for -n (replication) because we don't allow the user
    # to specify it here (only allowed in run-instances).
    additional_nodes_layout = NodeLayout(options)

    # In virtualized cluster deployments, we need to make sure that the user
    # has already set up SSH keys.
    if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
      for ip in options.ips.values():
        # throws a ShellException if the SSH key doesn't work
        RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)

    # Finally, find an AppController and send it a message to add
    # the given nodes with the new roles.
    AppScaleLogger.log("Sending request to add instances")
    login_ip = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_ip, LocalState.get_secret_key(
      options.keyname))
    acc.start_roles_on_nodes(json.dumps(options.ips))

    # TODO(cgb): Should we wait for the new instances to come up and get
    # initialized?
    AppScaleLogger.success("Successfully sent request to add instances " + \
      "to this AppScale deployment.")
Пример #9
0
    def terminate_virtualized_cluster(cls, keyname, is_verbose):
        """Stops all API services running on all nodes in the currently running
    AppScale deployment.

    Args:
      keyname: The name of the SSH keypair used for this AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands executed
        to stdout.
    """
        AppScaleLogger.log(
            "Terminating appscale deployment with keyname {0}".format(keyname))
        time.sleep(2)

        shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
        try:
            secret = LocalState.get_secret_key(keyname)
        except IOError:
            # We couldn't find the secret key: AppScale is most likely not
            # running.
            raise AppScaleException("Couldn't find AppScale secret key.")

        acc = AppControllerClient(shadow_host, secret)
        try:
            all_ips = acc.get_all_public_ips()
        except Exception as exception:
            AppScaleLogger.warn(
                'Saw Exception while getting deployments IPs {0}'.format(
                    str(exception)))
            all_ips = LocalState.get_all_public_ips(keyname)

        threads = []
        for ip in all_ips:
            thread = threading.Thread(target=cls.stop_remote_appcontroller,
                                      args=(ip, keyname, is_verbose))
            thread.start()
            threads.append(thread)

        for thread in threads:
            thread.join()

        boxes_shut_down = 0
        is_running_regex = re.compile("appscale-controller stop")
        for ip in all_ips:
            AppScaleLogger.log(
                "Shutting down AppScale API services at {0}".format(ip))
            while True:
                remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
                AppScaleLogger.verbose(remote_output, is_verbose)
                if not is_running_regex.match(remote_output):
                    break
                time.sleep(0.3)
            boxes_shut_down += 1

        if boxes_shut_down != len(all_ips):
            raise AppScaleException(
                "Couldn't terminate your AppScale deployment on"
                " all machines - please do so manually.")

        AppScaleLogger.log(
            "Terminated AppScale on {0} machines.".format(boxes_shut_down))
Пример #10
0
    def gather_logs(cls, options):
        """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        # First, make sure that the place we want to store logs doesn't
        # already exist.
        if os.path.exists(options.location):
            raise AppScaleException("Can't gather logs, as the location you " + \
              "specified, {0}, already exists.".format(options.location))

        acc = AppControllerClient(LocalState.get_login_host(options.keyname),
                                  LocalState.get_secret_key(options.keyname))

        # do the mkdir after we get the secret key, so that a bad keyname will
        # cause the tool to crash and not create this directory
        os.mkdir(options.location)

        for ip in acc.get_all_public_ips():
            # Get the logs from each node, and store them in our local directory
            local_dir = "{0}/{1}".format(options.location, ip)
            os.mkdir(local_dir)
            RemoteHelper.scp_remote_to_local(ip, options.keyname,
                                             '/var/log/appscale', local_dir,
                                             options.verbose)
        AppScaleLogger.success("Successfully copied logs to {0}".format(
            options.location))
 def test_set_deployment_id(self):
   host = 'boo'
   secret = 'baz'
   # The function should return whatever run_with_timeout_returns.
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return()
   acc = AppControllerClient(host, secret)
   acc.get_deployment_id()
 def test_set_deployment_id(self):
     host = 'boo'
     secret = 'baz'
     # The function should return whatever run_with_timeout_returns.
     flexmock(AppControllerClient).should_receive('run_with_timeout')\
       .and_return()
     acc = AppControllerClient(host, secret)
     acc.get_deployment_id()
Пример #13
0
  def terminate_virtualized_cluster(cls, keyname, is_verbose):
    """Stops all API services running on all nodes in the currently running
    AppScale deployment.

    Args:
      keyname: The name of the SSH keypair used for this AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands executed
        to stdout.
    """
    AppScaleLogger.log("Terminating appscale deployment with keyname {0}"
                       .format(keyname))
    time.sleep(2)

    shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
    try:
      secret = LocalState.get_secret_key(keyname)
    except IOError:
      # We couldn't find the secret key: AppScale is most likely not
      # running.
      raise AppScaleException("Couldn't find AppScale secret key.")

    acc = AppControllerClient(shadow_host, secret)
    try:
      all_ips = acc.get_all_public_ips()
    except Exception as exception:
      AppScaleLogger.warn('Saw Exception while getting deployments IPs {0}'.
                          format(str(exception)))
      all_ips = LocalState.get_all_public_ips(keyname)

    threads = []
    for ip in all_ips:
      thread = threading.Thread(target=cls.stop_remote_appcontroller, args=(ip,
        keyname, is_verbose))
      thread.start()
      threads.append(thread)

    for thread in threads:
      thread.join()

    boxes_shut_down = 0
    is_running_regex = re.compile("appscale-controller stop")
    for ip in all_ips:
      AppScaleLogger.log("Shutting down AppScale API services at {0}".
                         format(ip))
      while True:
        remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
        AppScaleLogger.verbose(remote_output, is_verbose)
        if not is_running_regex.match(remote_output):
          break
        time.sleep(0.3)
      boxes_shut_down += 1

    if boxes_shut_down != len(all_ips):
      raise AppScaleException("Couldn't terminate your AppScale deployment on"
                              " all machines - please do so manually.")

    AppScaleLogger.log("Terminated AppScale on {0} machines.".
                       format(boxes_shut_down))
 def test_get_deployment_id(self):
     # The function should return whatever run_with_timeout_returns.
     host = 'boo'
     secret = 'baz'
     deployment_id = 'foo'
     flexmock(AppControllerClient).should_receive('run_with_timeout')\
       .and_return(deployment_id)
     acc = AppControllerClient(host, secret)
     self.assertEqual(deployment_id, acc.get_deployment_id())
 def test_deployment_id_exists(self):
   # The function should return whatever run_with_timeout returns.
   host = 'boo'
   secret = 'baz'
   deployment_id_exists = True
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return(deployment_id_exists)
   acc = AppControllerClient(host, secret)
   self.assertEqual(deployment_id_exists, acc.deployment_id_exists())
 def test_get_deployment_id(self):
   # The function should return whatever run_with_timeout_returns.
   host = 'boo'
   secret = 'baz'
   deployment_id = 'foo'
   flexmock(AppControllerClient).should_receive('run_with_timeout')\
     .and_return(deployment_id)
   acc = AppControllerClient(host, secret)
   self.assertEqual(deployment_id, acc.get_deployment_id())
 def test_deployment_id_exists(self):
     # The function should return whatever run_with_timeout returns.
     host = 'boo'
     secret = 'baz'
     deployment_id_exists = True
     flexmock(AppControllerClient).should_receive('run_with_timeout')\
       .and_return(deployment_id_exists)
     acc = AppControllerClient(host, secret)
     self.assertEqual(deployment_id_exists, acc.deployment_id_exists())
Пример #18
0
  def create_user_accounts(cls, email, password, public_ip, keyname):
    """Registers two new user accounts with the UserAppServer.

    One account is the standard account that users log in with (via their
    e-mail address. The other is their XMPP account, so that they can log into
    any jabber-compatible service and send XMPP messages to their application
    (and receive them).

    Args:
      email: The e-mail address that should be registered for the user's
        standard account.
      password: The password that should be used for both the standard and XMPP
        accounts.
      public_ip: The location where the AppController can be found.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
    acc = AppControllerClient(public_ip, LocalState.get_secret_key(keyname))

    is_new_user = False
    # first, create the standard account
    encrypted_pass = LocalState.encrypt_password(email, password)
    if acc.does_user_exist(email):
      AppScaleLogger.log("User {0} already exists, so not creating it again.".
        format(email))
    else:
      acc.create_user(email, encrypted_pass)
      is_new_user = True

    # next, create the XMPP account. if the user's e-mail is [email protected], then that
    # means their XMPP account name is a@login_ip
    username_regex = re.compile('\A(.*)@')
    username = username_regex.match(email).groups()[0]

    try:
      login_host = acc.get_property('login')['login']
    except KeyError:
      raise AppControllerException('login property not found')

    xmpp_user = "******".format(username, login_host)
    xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

    is_xmpp_user_exist = acc.does_user_exist(xmpp_user)

    if is_xmpp_user_exist and is_new_user:
      AppScaleLogger.log("XMPP User {0} conflict!".format(xmpp_user))

      generated_xmpp_username = LocalState.generate_xmpp_username(username)
      xmpp_user = "******".format(generated_xmpp_username, login_host)
      xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

      acc.create_user(xmpp_user, xmpp_pass)
    elif is_xmpp_user_exist and not is_new_user:
      AppScaleLogger.log("XMPP User {0} already exists, so not creating it again.".format(xmpp_user))
    else:
      acc.create_user(xmpp_user, xmpp_pass)
    AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
Пример #19
0
    def remove_app(cls, options):
        """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if not options.confirm:
            response = raw_input(
                'Are you sure you want to remove this application? (y/N) ')
            if response.lower() not in ['y', 'yes']:
                raise AppScaleException("Cancelled application removal.")

        login_host = LocalState.get_login_host(options.keyname)
        secret = LocalState.get_secret_key(options.keyname)
        acc = AppControllerClient(login_host, secret)

        # Makes a call to the AppController to get all the stats and looks
        # through them for the http port the app can be reached on.
        http_port = None
        for _ in range(cls.MAX_RETRIES + 1):
            result = acc.get_all_stats()
            try:
                json_result = json.loads(result)
                apps_result = json_result['apps']
                current_app = apps_result[options.appname]
                http_port = current_app['http']
                if http_port:
                    break
                time.sleep(cls.SLEEP_TIME)
            except (KeyError, ValueError):
                AppScaleLogger.verbose(
                    "Got json error from get_all_data result.",
                    options.verbose)
                time.sleep(cls.SLEEP_TIME)
        if not http_port:
            raise AppScaleException(
                "Unable to get the serving port for the application.")

        acc.stop_app(options.appname)
        AppScaleLogger.log("Please wait for your app to shut down.")

        for _ in range(cls.MAX_RETRIES + 1):
            if RemoteHelper.is_port_open(login_host, http_port,
                                         options.verbose):
                time.sleep(cls.SLEEP_TIME)
                AppScaleLogger.log("Waiting for {0} to terminate...".format(
                    options.appname))
            else:
                AppScaleLogger.success("Done shutting down {0}.".format(
                    options.appname))
                return
        AppScaleLogger.warn("App {0} may still be running.".format(
            options.appname))
Пример #20
0
  def get_deployment_id(cls, head_node, keyname):
    """ Retrieve this AppScale deployment's ID.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
    """
    secret = LocalState.get_secret_key(keyname)
    acc = AppControllerClient(head_node, secret)
    return acc.get_deployment_id()
Пример #21
0
    def get_deployment_id(cls, head_node, keyname):
        """ Retrieve this AppScale deployment's ID.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
    """
        secret = LocalState.get_secret_key(keyname)
        acc = AppControllerClient(head_node, secret)
        return acc.get_deployment_id()
Пример #22
0
  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(options.location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {0}, already exists.".format(options.location))

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))

    try:
      all_ips = acc.get_all_public_ips()
    except socket.error:  # Occurs when the AppController has failed.
      AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
        "machines in this AppScale deployment. Using our locally cached " + \
        "info instead.")
      all_ips = LocalState.get_all_public_ips(options.keyname)

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(options.location)

    for ip in all_ips:
      # Get the logs from each node, and store them in our local directory
      local_dir = "{0}/{1}".format(options.location, ip)
      os.mkdir(local_dir)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
        local_dir, options.verbose)
      try:
        RemoteHelper.scp_remote_to_local(ip, options.keyname,
          '/var/log/cassandra', local_dir, options.verbose)
      except ShellException:
        pass

      try:
        RemoteHelper.scp_remote_to_local(ip, options.keyname,
          '/var/log/zookeeper', local_dir, options.verbose)
      except ShellException:
        pass

      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/kern.log',
        local_dir, options.verbose)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/syslog',
        local_dir, options.verbose)
    AppScaleLogger.success("Successfully copied logs to {0}".format(
      options.location))
Пример #23
0
    def set_deployment_id(cls, head_node, keyname, deployment_id):
        """ Set a deployment ID to use for communicating with the AppScale
    Portal.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
      deployment_id: A string containing the deployment ID.
    """
        secret = LocalState.get_secret_key(keyname)
        acc = AppControllerClient(head_node, secret)
        acc.set_deployment_id(deployment_id)
Пример #24
0
  def set_deployment_id(cls, head_node, keyname, deployment_id):
    """ Set a deployment ID to use for communicating with the AppScale
    Portal.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
      deployment_id: A string containing the deployment ID.
    """
    secret = LocalState.get_secret_key(keyname)
    acc = AppControllerClient(head_node, secret)
    acc.set_deployment_id(deployment_id)
Пример #25
0
  def update_local_metadata(cls, options, db_master, head_node):
    """Writes a locations.json file to the local filesystem,
    that the tools can use to locate machines in an AppScale deployment.

    Args:
      options: A Namespace that indicates deployment-specific parameters not
        relating to the placement strategy in use.
      db_master: A str representing the location of the database master.
      head_node: A str representing the location we can reach an
        AppController at.
    """
    # find out every machine's IP address and what they're doing
    acc = AppControllerClient(head_node, cls.get_secret_key(options.keyname))
    role_info = acc.get_role_info()

    infrastructure = options.infrastructure or 'xen'

    # write our yaml metadata file
    appscalefile_contents = {
      'infrastructure' : infrastructure,
      'group' : options.group,
    }

    if infrastructure != 'xen':
      appscalefile_contents['zone'] = options.zone

    if infrastructure == 'gce':
      appscalefile_contents['project'] = options.project
    elif infrastructure in ['ec2', 'euca']:
      appscalefile_contents['EC2_ACCESS_KEY'] = options.EC2_ACCESS_KEY
      appscalefile_contents['EC2_SECRET_KEY'] = options.EC2_SECRET_KEY
      appscalefile_contents['EC2_URL'] = options.EC2_URL
    elif infrastructure == 'azure':
      appscalefile_contents['azure_subscription_id'] = options.azure_subscription_id
      appscalefile_contents['azure_app_id'] = options.azure_app_id
      appscalefile_contents['azure_app_secret_key'] = options.azure_app_secret_key
      appscalefile_contents['azure_tenant_id'] = options.azure_tenant_id
      appscalefile_contents['azure_resource_group'] = options.azure_resource_group
      appscalefile_contents['azure_storage_account'] = options.azure_storage_account
      appscalefile_contents['azure_group_tag'] = options.azure_group_tag

    locations_json = {
      'node_info': role_info,
      'infrastructure_info': appscalefile_contents
    }

    # and now we can write the json metadata file
    with open(cls.get_locations_json_location(options.keyname), 'w') \
        as file_handle:
      file_handle.write(json.dumps(locations_json))
Пример #26
0
    def remove_app(cls, options):
        """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if not options.confirm:
            response = raw_input("Are you sure you want to remove this application? (y/N) ")
            if response.lower() not in ["y", "yes"]:
                raise AppScaleException("Cancelled application removal.")

        login_host = LocalState.get_login_host(options.keyname)
        secret = LocalState.get_secret_key(options.keyname)
        acc = AppControllerClient(login_host, secret)

        if not acc.is_app_running(options.appname):
            raise AppScaleException("The given application is not currently running.")

        # Makes a call to the AppController to get all the stats and looks
        # through them for the http port the app can be reached on.
        http_port = None
        for _ in range(cls.MAX_RETRIES + 1):
            result = acc.get_all_stats()
            try:
                json_result = json.loads(result)
                apps_result = json_result["apps"]
                current_app = apps_result[options.appname]
                http_port = current_app["http"]
                if http_port:
                    break
                time.sleep(cls.SLEEP_TIME)
            except (KeyError, ValueError):
                AppScaleLogger.verbose("Got json error from get_all_data result.", options.verbose)
                time.sleep(cls.SLEEP_TIME)
        if not http_port:
            raise AppScaleException("Unable to get the serving port for the application.")

        acc.stop_app(options.appname)
        AppScaleLogger.log("Please wait for your app to shut down.")

        for _ in range(cls.MAX_RETRIES + 1):
            if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
                time.sleep(cls.SLEEP_TIME)
                AppScaleLogger.log("Waiting for {0} to terminate...".format(options.appname))
            else:
                AppScaleLogger.success("Done shutting down {0}.".format(options.appname))
                return
        AppScaleLogger.warn("App {0} may still be running.".format(options.appname))
Пример #27
0
    def appscale_has_deployment_id(cls, head_node, keyname):
        """ Try to retrieve a deployment ID from ZooKeeper to see if this
    deployment has already been registered.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
    Returns:
      A boolean indicating whether the deployment ID exists or not.
    """
        # Check if head node has a deployment ID stored.
        secret = LocalState.get_secret_key(keyname)
        acc = AppControllerClient(head_node, secret)
        return acc.deployment_id_exists()
    def set_property(cls, options):
        """Instructs AppScale to replace the value it uses for a particular
    AppController instance variable (property) with a new value.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    """
        shadow_host = LocalState.get_host_with_role(options.keyname, "shadow")
        acc = AppControllerClient(shadow_host, LocalState.get_secret_key(options.keyname))
        result = acc.set_property(options.property_name, options.property_value)
        if result == "OK":
            AppScaleLogger.success("Successfully updated the given property.")
        else:
            raise AppControllerException("Unable to update the given property " + "because: {0}".format(result))
Пример #29
0
  def appscale_has_deployment_id(cls, head_node, keyname):
    """ Try to retrieve a deployment ID from ZooKeeper to see if this
    deployment has already been registered.

    Args:
      head_node: A string containing the IP address of the head node.
      keyname: A string representing the SSH keypair name used for this
        AppScale deployment.
    Returns:
      A boolean indicating whether the deployment ID exists or not.
    """
    # Check if head node has a deployment ID stored.
    secret = LocalState.get_secret_key(keyname)
    acc = AppControllerClient(head_node, secret)
    return acc.deployment_id_exists()
    def get_property(cls, options):
        """Queries AppScale for a list of system properties matching the provided
    regular expression, as well as the values associated with each matching
    property.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Returns:
      A dict mapping each property matching the given regex to its associated
      value.
    """
        shadow_host = LocalState.get_host_with_role(options.keyname, "shadow")
        acc = AppControllerClient(shadow_host, LocalState.get_secret_key(options.keyname))

        return acc.get_property(options.property)
Пример #31
0
    def update_local_metadata(cls, options, node_layout, host, instance_id):
        """Writes a locations.yaml and locations.json file to the local filesystem,
    that the tools can use to locate machines in an AppScale deployment.

    Args:
      options: A Namespace that indicates deployment-specific parameters not
        relating to the placement strategy in use.
      node_layout: A NodeLayout that indicates the placement strategy in use
        for this deployment.
      host: A str representing the location we can reach an AppController at.
      instance_id: The instance ID (if running in a cloud environment)
        associated with the given host.
    """
        # find out every machine's IP address and what they're doing
        acc = AppControllerClient(host, cls.get_secret_key(options.keyname))
        all_ips = [str(ip) for ip in acc.get_all_public_ips()]
        role_info = acc.get_role_info()

        infrastructure = options.infrastructure or 'xen'

        # write our yaml metadata file
        yaml_contents = {
            'load_balancer': str(host),
            'instance_id': str(instance_id),
            'table': options.table,
            'secret': cls.get_secret_key(options.keyname),
            'db_master': node_layout.db_master().public_ip,
            'ips': all_ips,
            'infrastructure': infrastructure,
            'group': options.group,
        }

        if infrastructure != "xen":
            yaml_contents['zone'] = options.zone

        if infrastructure == "gce":
            yaml_contents['project'] = options.project

        with open(cls.get_locations_yaml_location(options.keyname), 'w') \
            as file_handle:
            file_handle.write(
                yaml.dump(yaml_contents, default_flow_style=False))

        # and now we can write the json metadata file
        with open(cls.get_locations_json_location(options.keyname), 'w') \
            as file_handle:
            file_handle.write(json.dumps(role_info))
Пример #32
0
  def update_local_metadata(cls, options, db_master, head_node):
    """Writes a locations.json file to the local filesystem,
    that the tools can use to locate machines in an AppScale deployment.

    Args:
      options: A Namespace that indicates deployment-specific parameters not
        relating to the placement strategy in use.
      db_master: A str representing the location of the database master.
      head_node: A str representing the location we can reach an
        AppController at.
    """
    # find out every machine's IP address and what they're doing
    acc = AppControllerClient(head_node, cls.get_secret_key(options.keyname))
    role_info = acc.get_role_info()

    infrastructure = options.infrastructure or 'xen'

    # write our yaml metadata file
    appscalefile_contents = {
      'infrastructure' : infrastructure,
      'group' : options.group,
    }

    if infrastructure != "xen":
      appscalefile_contents['zone'] = options.zone

    if infrastructure == "gce":
      appscalefile_contents['project'] = options.project

    if infrastructure == 'azure':
      appscalefile_contents['azure_subscription_id'] = options.azure_subscription_id
      appscalefile_contents['azure_app_id'] = options.azure_app_id
      appscalefile_contents['azure_app_secret_key'] = options.azure_app_secret_key
      appscalefile_contents['azure_tenant_id'] = options.azure_tenant_id
      appscalefile_contents['azure_resource_group'] = options.azure_resource_group
      appscalefile_contents['azure_storage_account'] = options.azure_storage_account
      appscalefile_contents['azure_group_tag'] = options.azure_group_tag

    locations_json = {
      'node_info': role_info,
      'infrastructure_info': appscalefile_contents
    }

    # and now we can write the json metadata file
    with open(cls.get_locations_json_location(options.keyname), 'w') \
        as file_handle:
      file_handle.write(json.dumps(locations_json))
  def update_local_metadata(cls, options, node_layout, host, instance_id):
    """Writes a locations.yaml and locations.json file to the local filesystem,
    that the tools can use to locate machines in an AppScale deployment.

    Args:
      options: A Namespace that indicates deployment-specific parameters not
        relating to the placement strategy in use.
      node_layout: A NodeLayout that indicates the placement strategy in use
        for this deployment.
      host: A str representing the location we can reach an AppController at.
      instance_id: The instance ID (if running in a cloud environment)
        associated with the given host.
    """
    # find out every machine's IP address and what they're doing
    acc = AppControllerClient(host, cls.get_secret_key(options.keyname))
    all_ips = [str(ip) for ip in acc.get_all_public_ips()]
    role_info = acc.get_role_info()

    infrastructure = options.infrastructure or 'xen'

    # write our yaml metadata file
    yaml_contents = {
      'load_balancer' : str(host),
      'instance_id' : str(instance_id),
      'table' : options.table,
      'secret' : cls.get_secret_key(options.keyname),
      'db_master' : node_layout.db_master().public_ip,
      'ips' : all_ips,
      'infrastructure' : infrastructure,
      'group' : options.group,
    }

    if infrastructure != "xen":
      yaml_contents['zone'] = options.zone

    if infrastructure == "gce":
      yaml_contents['project'] = options.project

    with open(cls.get_locations_yaml_location(options.keyname), 'w') \
        as file_handle:
      file_handle.write(yaml.dump(yaml_contents, default_flow_style=False))

    # and now we can write the json metadata file
    with open(cls.get_locations_json_location(options.keyname), 'w') \
        as file_handle:
      file_handle.write(json.dumps(role_info))
Пример #34
0
  def set_property(cls, options):
    """Instructs AppScale to replace the value it uses for a particular
    AppController instance variable (property) with a new value.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    """
    shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow')
    acc = AppControllerClient(shadow_host, LocalState.get_secret_key(
      options.keyname))
    result = acc.set_property(options.property_name, options.property_value)
    if result == 'OK':
      AppScaleLogger.success("Successfully updated the given property.")
    else:
      raise AppControllerException("Unable to update the given property " +
        "because: {0}".format(result))
Пример #35
0
    def get_property(cls, options):
        """Queries AppScale for a list of system properties matching the provided
    regular expression, as well as the values associated with each matching
    property.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Returns:
      A dict mapping each property matching the given regex to its associated
      value.
    """
        shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow')
        acc = AppControllerClient(shadow_host,
                                  LocalState.get_secret_key(options.keyname))

        return acc.get_property(options.property)
Пример #36
0
  def wait_for_machines_to_finish_loading(cls, host, keyname):
    """Queries all of the AppControllers in this AppScale deployment to see if
    they have started all of the API services on their machine, and if not,
    waits until they have.

    Args:
      host: The location where an AppController can be found, who will then have
        the locations of all the other AppControllers in this AppScale
        deployment.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
    acc = AppControllerClient(host, LocalState.get_secret_key(keyname))
    while True:
      if acc.is_initialized():
        break
      else:
        time.sleep(cls.WAIT_TIME)
Пример #37
0
    def describe_instances(cls, options):
        """Queries each node in the currently running AppScale deployment and
    reports on their status.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        login_host = LocalState.get_login_host(options.keyname)
        login_acc = AppControllerClient(
            login_host, LocalState.get_secret_key(options.keyname))

        for ip in login_acc.get_all_public_ips():
            acc = AppControllerClient(
                ip, LocalState.get_secret_key(options.keyname))
            AppScaleLogger.log("Status of node at {0}:".format(ip))
            try:
                AppScaleLogger.log(acc.get_status())
            except Exception as exception:
                AppScaleLogger.warn("Unable to contact machine: {0}\n".format(
                    str(exception)))

        AppScaleLogger.success("View status information about your AppScale " + \
          "deployment at http://{0}:{1}/status".format(login_host,
          RemoteHelper.APP_DASHBOARD_PORT))
Пример #38
0
    def reset_password(cls, options):
        """Resets a user's password the currently running AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        secret = LocalState.get_secret_key(options.keyname)
        login_host = LocalState.get_login_host(options.keyname)
        username, password = LocalState.get_credentials(is_admin=False)
        encrypted_password = LocalState.encrypt_password(username, password)

        acc = AppControllerClient(login_host, secret)

        try:
            acc.reset_password(username, encrypted_password)
            AppScaleLogger.success("The password was successfully changed for the " \
              "given user.")
        except Exception as exception:
            AppScaleLogger.warn("Could not change the user's password for the " + \
              "following reason: {0}".format(str(exception)))
            sys.exit(1)
Пример #39
0
  def describe_instances(cls, options):
    """Queries each node in the currently running AppScale deployment and
    reports on their status.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    login_host = LocalState.get_login_host(options.keyname)
    login_acc = AppControllerClient(login_host,
      LocalState.get_secret_key(options.keyname))

    for ip in login_acc.get_all_public_ips():
      acc = AppControllerClient(ip, LocalState.get_secret_key(options.keyname))
      AppScaleLogger.log("Status of node at {0}:".format(ip))
      try:
        AppScaleLogger.log(acc.get_status())
      except Exception as e:
        AppScaleLogger.warn("Unable to contact machine: {0}\n".format(str(e)))

    AppScaleLogger.success("View status information about your AppScale " + \
      "deployment at http://{0}/status".format(login_host))
Пример #40
0
  def reset_password(cls, options):
    """Resets a user's password the currently running AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    secret = LocalState.get_secret_key(options.keyname)
    login_host = LocalState.get_login_host(options.keyname)
    username, password = LocalState.get_credentials(is_admin=False)
    encrypted_password = LocalState.encrypt_password(username, password)

    acc = AppControllerClient(login_host,secret)

    try:
      acc.reset_password(username, encrypted_password)
      AppScaleLogger.success("The password was successfully changed for the " \
        "given user.")
    except Exception as exception:
      AppScaleLogger.warn("Could not change the user's password for the " + \
        "following reason: {0}".format(str(exception)))
      sys.exit(1)
Пример #41
0
    def relocate_app(cls, options):
        """Instructs AppScale to move the named application to a different port.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Raises:
      AppScaleException: If the named application isn't running in this AppScale
        cloud, if the destination port is in use by a different application, or
        if the AppController rejects the request to relocate the application (in
        which case it includes the reason why the rejection occurred).
    """
        login_host = LocalState.get_login_host(options.keyname)
        acc = AppControllerClient(login_host,
                                  LocalState.get_secret_key(options.keyname))

        app_info_map = acc.get_app_info_map()
        if options.appname not in app_info_map.keys():
            raise AppScaleException("The given application, {0}, is not currently " \
              "running in this AppScale cloud, so we can't move it to a different " \
              "port.".format(options.appname))

        relocate_result = acc.relocate_app(options.appname, options.http_port,
                                           options.https_port)
        if relocate_result == "OK":
            AppScaleLogger.success("Successfully issued request to move {0} to " \
              "ports {1} and {2}.".format(options.appname, options.http_port,
              options.https_port))
            RemoteHelper.sleep_until_port_is_open(login_host,
                                                  options.http_port,
                                                  options.verbose)
            AppScaleLogger.success(
                "Your app serves unencrypted traffic at: " +
                "http://{0}:{1}".format(login_host, options.http_port))
            AppScaleLogger.success(
                "Your app serves encrypted traffic at: " +
                "https://{0}:{1}".format(login_host, options.https_port))
        else:
            raise AppScaleException(relocate_result)
Пример #42
0
    def create_user_accounts(cls, email, password, public_ip, keyname):
        """Registers two new user accounts with the UserAppServer.

    One account is the standard account that users log in with (via their
    e-mail address. The other is their XMPP account, so that they can log into
    any jabber-compatible service and send XMPP messages to their application
    (and receive them).

    Args:
      email: The e-mail address that should be registered for the user's
        standard account.
      password: The password that should be used for both the standard and XMPP
        accounts.
      public_ip: The location where the AppController can be found.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
        acc = AppControllerClient(public_ip,
                                  LocalState.get_secret_key(keyname))

        # first, create the standard account
        encrypted_pass = LocalState.encrypt_password(email, password)
        if acc.does_user_exist(email):
            AppScaleLogger.log(
                "User {0} already exists, so not creating it again.".format(
                    email))
        else:
            acc.create_user(email, encrypted_pass)

        # next, create the XMPP account. if the user's e-mail is [email protected], then that
        # means their XMPP account name is a@login_ip
        username_regex = re.compile('\A(.*)@')
        username = username_regex.match(email).groups()[0]
        xmpp_user = "******".format(username,
                                     LocalState.get_login_host(keyname))
        xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

        if acc.does_user_exist(xmpp_user):
            AppScaleLogger.log(
                "XMPP User {0} already exists, so not creating it again.".
                format(xmpp_user))
        else:
            acc.create_user(xmpp_user, xmpp_pass)
        AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
Пример #43
0
  def create_user_accounts(cls, email, password, public_ip, keyname,
    clear_datastore):
    """Registers two new user accounts with the UserAppServer.

    One account is the standard account that users log in with (via their
    e-mail address. The other is their XMPP account, so that they can log into
    any jabber-compatible service and send XMPP messages to their application
    (and receive them).

    Args:
      email: The e-mail address that should be registered for the user's
        standard account.
      password: The password that should be used for both the standard and XMPP
        accounts.
      public_ip: The location where the AppController can be found.
      keyname: The name of the SSH keypair used for this AppScale deployment.
      clear_datastore: A bool that indicates if we expect the datastore to be
        emptied, and thus not contain any user accounts.
    """
    acc = AppControllerClient(public_ip, LocalState.get_secret_key(keyname))

    # first, create the standard account
    encrypted_pass = LocalState.encrypt_password(email, password)
    if not clear_datastore and acc.does_user_exist(email):
      AppScaleLogger.log("User {0} already exists, so not creating it again.".
        format(email))
    else:
      acc.create_user(email, encrypted_pass)

    # next, create the XMPP account. if the user's e-mail is [email protected], then that
    # means their XMPP account name is a@login_ip
    username_regex = re.compile('\A(.*)@')
    username = username_regex.match(email).groups()[0]
    xmpp_user = "******".format(username, LocalState.get_login_host(keyname))
    xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

    if not clear_datastore and acc.does_user_exist(xmpp_user):
      AppScaleLogger.log(
        "XMPP User {0} already exists, so not creating it again.".
        format(xmpp_user))
    else:
      acc.create_user(xmpp_user, xmpp_pass)
    AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
  def test_appscale_in_one_node_virt_deployment_with_login_override(self):
    # let's say that appscale isn't already running
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('update_local_metadata').and_return()
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "1.2.3.4",
        "private_ip" : "1.2.3.4",
        "jobs" : ["shadow", "login"]
      }])))
    self.local_state.should_receive('get_secret_key').and_return("fookey")

    flexmock(RemoteHelper)
    RemoteHelper.should_receive('start_head_node')\
        .and_return(('1.2.3.4','i-ABCDEFG'))
    RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
    RemoteHelper.should_receive('copy_local_metadata').and_return()
    RemoteHelper.should_receive('create_user_accounts').and_return()
    RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
        .and_return()
    RemoteHelper.should_receive('copy_deployment_credentials')

    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)
    AppControllerClient.should_receive('is_initialized').and_return(True)
    AppControllerClient.should_receive('set_admin_role').and_return()

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine:  1.2.3.4
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test",
      "--login_host", "www.booscale.com"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
Пример #45
0
  def wait_for_machines_to_finish_loading(cls, host, keyname):
    """Queries all of the AppControllers in this AppScale deployment to see if
    they have started all of the API services on their machine, and if not,
    waits until they have.

    Args:
      host: The location where an AppController can be found, who will then have
        the locations of all the other AppControllers in this AppScale
        deployment.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
    acc = AppControllerClient(host, LocalState.get_secret_key(keyname))
    all_ips = acc.get_all_public_ips()

    for ip in all_ips:
      while True:
        acc = AppControllerClient(ip, LocalState.get_secret_key(keyname))
        if acc.is_initialized():
          break
        else:
          time.sleep(cls.WAIT_TIME)
Пример #46
0
    def start_head_node(cls, options, my_id, node_layout):
        """Starts the first node in an AppScale deployment and instructs it to start
    API services on its own node, as well as the other nodes in the deployment.

    This includes spawning the first node in the deployment, copying over all
    deployment-specific files to it, and starting its AppController service.

    Args:
      options: A Namespace that includes parameters passed in by the user that
        define non-placement-strategy-related deployment options (e.g., keypair
        names, security group names).
      my_id: A str that is used to uniquely identify this AppScale deployment
        with the remote start application.
      node_layout: A NodeLayout that describes the placement strategy that
        should be used for this AppScale deployment.
    Returns:
      The public IP and instance ID (a dummy value in non-cloud deployments)
      corresponding to the node that was started.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        The message in this exception indicates why the crash occurred.
    """
        secret_key = LocalState.generate_secret_key(options.keyname)
        AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
                               options.verbose)

        if options.infrastructure:
            instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(
                options)
        else:
            instance_id = cls.DUMMY_INSTANCE_ID
            public_ip = node_layout.head_node().public_ip
            private_ip = node_layout.head_node().private_ip

        AppScaleLogger.log(
            "Log in to your head node: ssh -i {0} root@{1}".format(
                LocalState.get_key_path_from_name(options.keyname), public_ip))

        try:
            cls.ensure_machine_is_compatible(public_ip, options.keyname,
                                             options.table, options.verbose)
        except AppScaleException as ase:
            # On failure shutdown the cloud instances, cleanup the keys, but only
            # if --test is not set.
            if options.infrastructure:
                if not options.test:
                    try:
                        cls.terminate_cloud_instance(instance_id, options)
                    except Exception as tcie:
                        AppScaleLogger.log(
                            "Error terminating instances: {0}".format(
                                str(tcie)))
                raise AppScaleException("{0} Please ensure that the "\
                  "image {1} has AppScale {2} installed on it."
                  .format(str(ase), options.machine, APPSCALE_VERSION))
            else:
                raise AppScaleException("{0} Please login to that machine and ensure "\
                  "that AppScale {1} is installed on it."
                  .format(str(ase), APPSCALE_VERSION))

        if options.scp:
            AppScaleLogger.log(
                "Copying over local copy of AppScale from {0}".format(
                    options.scp))
            cls.rsync_files(public_ip, options.keyname, options.scp,
                            options.verbose)

        # On Euca, we've seen issues where attaching the EBS volume right after
        # the instance starts doesn't work. This sleep lets the instance fully
        # start up and get volumes attached to it correctly.
        if options.infrastructure and options.infrastructure == 'euca' and \
          options.disks:
            time.sleep(30)

        if options.infrastructure:
            agent = InfrastructureAgentFactory.create_agent(
                options.infrastructure)
            params = agent.get_params_from_args(options)
            additional_params = {}

            if agent.PARAM_CREDENTIALS in params:
                additional_params = params[agent.PARAM_CREDENTIALS]

            if options.use_spot_instances:
                additional_params[agent.PARAM_SPOT_PRICE] = \
                  str(params[agent.PARAM_SPOT_PRICE])

            if agent.PARAM_REGION in params:
                additional_params[agent.PARAM_REGION] = params[
                    agent.PARAM_REGION]
        else:
            additional_params = {}

        deployment_params = LocalState.generate_deployment_params(
            options, node_layout, public_ip, additional_params)
        AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
                               options.verbose)
        AppScaleLogger.log(
            "Head node successfully initialized at {0}.".format(public_ip))

        AppScaleLogger.remote_log_tools_state(options, my_id,
                                              "started head node",
                                              APPSCALE_VERSION)
        time.sleep(10)  # gives machines in cloud extra time to boot up

        cls.copy_deployment_credentials(public_ip, options)
        cls.run_user_commands(public_ip, options.user_commands,
                              options.keyname, options.verbose)
        cls.start_remote_appcontroller(public_ip, options.keyname,
                                       options.verbose)

        acc = AppControllerClient(public_ip, secret_key)
        locations = [{
            'public_ip': public_ip,
            'private_ip': private_ip,
            'jobs': node_layout.head_node().roles,
            'instance_id': instance_id,
            'disk': node_layout.head_node().disk
        }]
        try:
            acc.set_parameters(locations,
                               LocalState.map_to_array(deployment_params))
        except Exception as exception:
            AppScaleLogger.warn('Saw Exception while setting AC parameters: {0}' \
              .format(str(exception)))
            message = RemoteHelper.collect_appcontroller_crashlog(
                public_ip, options.keyname, options.verbose)
            raise AppControllerException(message)

        return public_ip, instance_id
Пример #47
0
    def upload_app(cls, options):
        """Uploads the given App Engine application into AppScale.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Returns:
      A tuple containing the host and port where the application is serving
        traffic from.
    """
        if cls.TAR_GZ_REGEX.search(options.file):
            file_location = LocalState.extract_app_to_dir(
                options.file, options.verbose)
            created_dir = True
        else:
            file_location = options.file
            created_dir = False

        app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
        app_language = AppEngineHelper.get_app_runtime_from_app_config(
            file_location)
        AppEngineHelper.validate_app_id(app_id)

        acc = AppControllerClient(LocalState.get_login_host(options.keyname),
                                  LocalState.get_secret_key(options.keyname))
        userappserver_host = acc.get_uaserver_host(options.verbose)
        userappclient = UserAppClient(
            userappserver_host, LocalState.get_secret_key(options.keyname))

        if options.test:
            username = LocalState.DEFAULT_USER
        elif options.email:
            username = options.email
        else:
            username = LocalState.get_username_from_stdin(is_admin=False)

        if not userappclient.does_user_exist(username):
            password = LocalState.get_password_from_stdin()
            RemoteHelper.create_user_accounts(username, password,
                                              userappserver_host,
                                              options.keyname)

        app_exists = userappclient.does_app_exist(app_id)
        app_admin = userappclient.get_app_admin(app_id)
        if app_admin is not None and username != app_admin:
            raise AppScaleException("The given user doesn't own this application" + \
              ", so they can't upload an app with that application ID. Please " + \
              "change the application ID and try again.")

        if app_exists:
            AppScaleLogger.log(
                "Uploading new version of app {0}".format(app_id))
        else:
            AppScaleLogger.log(
                "Uploading initial version of app {0}".format(app_id))
            userappclient.reserve_app_id(username, app_id, app_language)

        remote_file_path = RemoteHelper.copy_app_to_host(
            file_location, options.keyname, options.verbose)

        acc.done_uploading(app_id, remote_file_path)
        acc.update([app_id])

        # now that we've told the AppController to start our app, find out what port
        # the app is running on and wait for it to start serving
        AppScaleLogger.log("Please wait for your app to start serving.")

        if app_exists:
            time.sleep(20)  # give the AppController time to restart the app

        serving_host, serving_port = userappclient.get_serving_info(
            app_id, options.keyname)
        RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
                                              options.verbose)
        AppScaleLogger.success(
            "Your app can be reached at the following URL: " +
            "http://{0}:{1}".format(serving_host, serving_port))

        if created_dir:
            shutil.rmtree(file_location)

        return (serving_host, serving_port)
Пример #48
0
    def gather_logs(cls, options):
        """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        # First, make sure that the place we want to store logs doesn't
        # already exist.
        if os.path.exists(options.location):
            raise AppScaleException("Can't gather logs, as the location you " + \
              "specified, {0}, already exists.".format(options.location))

        acc = AppControllerClient(LocalState.get_login_host(options.keyname),
                                  LocalState.get_secret_key(options.keyname))

        try:
            all_ips = acc.get_all_public_ips()
        except socket.error:  # Occurs when the AppController has failed.
            AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
              "machines in this AppScale deployment. Using our locally cached " + \
              "info instead.")
            all_ips = LocalState.get_all_public_ips(options.keyname)

        # do the mkdir after we get the secret key, so that a bad keyname will
        # cause the tool to crash and not create this directory
        os.mkdir(options.location)

        # The log paths that we collect logs from.
        log_paths = [
            '/var/log/appscale', '/var/log/kern.log*', '/var/log/monit.log*',
            '/var/log/nginx', '/var/log/syslog*', '/var/log/zookeeper'
        ]

        failures = False
        for ip in all_ips:
            # Get the logs from each node, and store them in our local directory
            local_dir = "{0}/{1}".format(options.location, ip)
            os.mkdir(local_dir)

            for log_path in log_paths:
                try:
                    RemoteHelper.scp_remote_to_local(ip, options.keyname,
                                                     log_path, local_dir,
                                                     options.verbose)
                except ShellException as shell_exception:
                    failures = True
                    AppScaleLogger.warn(
                        "Unable to collect logs from '{}' for host '{}'".
                        format(log_path, ip))
                    AppScaleLogger.verbose(
                        "Encountered exception: {}".format(
                            str(shell_exception)), options.verbose)

        if failures:
            AppScaleLogger.log(
                "Done copying to {0}. There were "
                "failures while collecting AppScale logs.".format(
                    options.location))
        else:
            AppScaleLogger.success(
                "Successfully collected all AppScale logs into "
                "{0}".format(options.location))
Пример #49
0
    def run_instances(cls, options):
        """Starts a new AppScale deployment with the parameters given.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        When this occurs, the message in the exception contains the reason why
        the AppController crashed.
      BadConfigurationException: If the user passes in options that are not
        sufficient to start an AppScale deployment (e.g., running on EC2 but
        not specifying the AMI to use), or if the user provides us
        contradictory options (e.g., running on EC2 but not specifying EC2
        credentials).
    """
        LocalState.make_appscale_directory()
        LocalState.ensure_appscale_isnt_running(options.keyname, options.force)

        if options.infrastructure:
            if not options.disks and not options.test and not options.force:
                LocalState.ensure_user_wants_to_run_without_disks()
            AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
                               " over the " + options.infrastructure +
                               " cloud.")
        else:
            AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
                               " over a virtualized cluster.")
        my_id = str(uuid.uuid4())
        AppScaleLogger.remote_log_tools_state(options, my_id, "started",
                                              APPSCALE_VERSION)

        node_layout = NodeLayout(options)
        if not node_layout.is_valid():
            raise BadConfigurationException("There were errors with your " + \
              "placement strategy:\n{0}".format(str(node_layout.errors())))

        if not node_layout.is_supported():
            AppScaleLogger.warn("Warning: This deployment strategy is not " + \
              "officially supported.")

        public_ip, instance_id = RemoteHelper.start_head_node(
            options, my_id, node_layout)
        AppScaleLogger.log(
            "\nPlease wait for AppScale to prepare your machines " +
            "for use.")

        # Write our metadata as soon as possible to let users SSH into those
        # machines via 'appscale ssh'
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        acc = AppControllerClient(public_ip,
                                  LocalState.get_secret_key(options.keyname))
        try:
            uaserver_host = acc.get_uaserver_host(options.verbose)
        except Exception:
            message = RemoteHelper.collect_appcontroller_crashlog(
                public_ip, options.keyname, options.verbose)
            raise AppControllerException(message)

        RemoteHelper.sleep_until_port_is_open(uaserver_host,
                                              UserAppClient.PORT,
                                              options.verbose)

        # Update our metadata again so that users can SSH into other boxes that
        # may have been started.
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        AppScaleLogger.log("UserAppServer is at {0}".format(uaserver_host))

        uaserver_client = UserAppClient(
            uaserver_host, LocalState.get_secret_key(options.keyname))

        if options.admin_user and options.admin_pass:
            AppScaleLogger.log("Using the provided admin username/password")
            username, password = options.admin_user, options.admin_pass
        elif options.test:
            AppScaleLogger.log("Using default admin username/password")
            username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
        else:
            username, password = LocalState.get_credentials()

        RemoteHelper.create_user_accounts(username, password, uaserver_host,
                                          options.keyname,
                                          options.clear_datastore)
        uaserver_client.set_admin_role(username)

        RemoteHelper.wait_for_machines_to_finish_loading(
            public_ip, options.keyname)
        # Finally, update our metadata once we know that all of the machines are
        # up and have started all their API services.
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        RemoteHelper.sleep_until_port_is_open(
            LocalState.get_login_host(options.keyname),
            RemoteHelper.APP_DASHBOARD_PORT, options.verbose)
        AppScaleLogger.success("AppScale successfully started!")
        AppScaleLogger.success("View status information about your AppScale " + \
          "deployment at http://{0}:{1}/status".format(LocalState.get_login_host(
          options.keyname), RemoteHelper.APP_DASHBOARD_PORT))
        AppScaleLogger.remote_log_tools_state(options, my_id, "finished",
                                              APPSCALE_VERSION)
Пример #50
0
  def upload_app(cls, options):
    """Uploads the given App Engine application into AppScale.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if cls.TAR_GZ_REGEX.search(options.file):
      file_location = LocalState.extract_app_to_dir(options.file,
        options.verbose)
      created_dir = True
    else:
      file_location = options.file
      created_dir = False

    app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
    app_language = AppEngineHelper.get_app_runtime_from_app_config(
      file_location)
    AppEngineHelper.validate_app_id(app_id)

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))
    userappserver_host = acc.get_uaserver_host(options.verbose)
    userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
      options.keyname))

    if options.test:
      username = LocalState.DEFAULT_USER
    elif options.email:
      username = options.email
    else:
      username = LocalState.get_username_from_stdin(is_admin=False)

    if not userappclient.does_user_exist(username):
      password = LocalState.get_password_from_stdin()
      RemoteHelper.create_user_accounts(username, password, userappserver_host,
        options.keyname)

    if userappclient.does_app_exist(app_id):
      raise AppScaleException("The given application is already running in " + \
        "AppScale. Please choose a different application ID or use " + \
        "appscale-remove-app to take down the existing application.")

    app_admin = userappclient.get_app_admin(app_id)
    if app_admin is not None and username != app_admin:
      raise AppScaleException("The given user doesn't own this application" + \
        ", so they can't upload an app with that application ID. Please " + \
        "change the application ID and try again.")

    AppScaleLogger.log("Uploading {0}".format(app_id))
    userappclient.reserve_app_id(username, app_id, app_language)

    remote_file_path = RemoteHelper.copy_app_to_host(file_location,
      options.keyname, options.verbose)
    acc.done_uploading(app_id, remote_file_path)
    acc.update([app_id])

    # now that we've told the AppController to start our app, find out what port
    # the app is running on and wait for it to start serving
    AppScaleLogger.log("Please wait for your app to start serving.")
    serving_host, serving_port = userappclient.get_serving_info(app_id,
      options.keyname)
    RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
      options.verbose)
    AppScaleLogger.success("Your app can be reached at the following URL: " +
      "http://{0}:{1}".format(serving_host, serving_port))

    if created_dir:
      shutil.rmtree(file_location)
  def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
    # let's say that appscale isn't already running
    local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
      os.sep + self.keyname + ".key"
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()
    self.local_state.should_receive('get_key_path_from_name').and_return(
      local_appscale_path)

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    self.setup_ec2_mocks()

    # also mock out acquiring a spot instance
    self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
      'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
      instance_type='m3.medium', count=1, placement='my-zone-1b')

    # assume that root login is not enabled
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)

    # assume that we can enable root login
    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo touch /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin='mktemp').and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5,
      stdin=re.compile(
        'sudo sed -n '
        '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
      )
    ).and_return()

    self.local_state.should_receive('shell').with_args(
      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
    ).and_return()

    # and assume that we can copy over our ssh keys fine
    self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
      False, 5).and_return()
    self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
      .format(self.keyname)), False, 5).and_return()

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell').with_args(re.compile('openssl'),
      False, stdin=None)

    # assume that we started monit fine
    self.local_state.should_receive('shell').with_args(re.compile('ssh'),
      False, 5, stdin=re.compile('monit'))

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('controller-17443.cfg'))

    self.setup_socket_mocks('public1')
    self.setup_appcontroller_mocks('public1', 'private1')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "public1",
        "private_ip" : "private1",
        "jobs" : ["shadow", "login"]
      }])))

    # copying over the locations yaml and json files should be fine
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))

    # same for the secret key
    self.local_state.should_receive('shell').with_args(re.compile('scp'),
      False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()

    flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    argv = [
      "--min", "1",
      "--max", "1",
      "--infrastructure", "ec2",
      "--machine", "ami-ABCDEFG",
      "--use_spot_instances",
      "--max_spot_price", "1.23",
      "--keyname", self.keyname,
      "--group", self.group,
      "--test",
      "--zone", "my-zone-1b"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)
Пример #52
0
    def upload_app(cls, options):
        """Uploads the given App Engine application into AppScale.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Returns:
      A tuple containing the host and port where the application is serving
        traffic from.
    """
        if cls.TAR_GZ_REGEX.search(options.file):
            file_location = LocalState.extract_tgz_app_to_dir(
                options.file, options.verbose)
            created_dir = True
        elif cls.ZIP_REGEX.search(options.file):
            file_location = LocalState.extract_zip_app_to_dir(
                options.file, options.verbose)
            created_dir = True
        elif os.path.isdir(options.file):
            file_location = options.file
            created_dir = False
        else:
            raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
              'or a directory. Please try uploading either a tar.gz file, a zip ' \
              'file, or a directory.'.format(options.file))

        try:
            app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
        except AppEngineConfigException:
            # Java App Engine users may have specified their war directory. In that
            # case, just move up one level, back to the app's directory.
            file_location = file_location + os.sep + ".."
            app_id = AppEngineHelper.get_app_id_from_app_config(file_location)

        app_language = AppEngineHelper.get_app_runtime_from_app_config(
            file_location)
        AppEngineHelper.validate_app_id(app_id)

        if app_language == 'java':
            if AppEngineHelper.is_sdk_mismatch(file_location):
                AppScaleLogger.warn(
                    'AppScale did not find the correct SDK jar ' +
                    'versions in your app. The current supported ' +
                    'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION +
                    '.')

        acc = AppControllerClient(LocalState.get_login_host(options.keyname),
                                  LocalState.get_secret_key(options.keyname))
        userappserver_host = acc.get_uaserver_host(options.verbose)
        userappclient = UserAppClient(
            userappserver_host, LocalState.get_secret_key(options.keyname))

        if options.test:
            username = LocalState.DEFAULT_USER
        elif options.email:
            username = options.email
        else:
            username = LocalState.get_username_from_stdin(is_admin=False)

        if not userappclient.does_user_exist(username):
            password = LocalState.get_password_from_stdin()
            RemoteHelper.create_user_accounts(username,
                                              password,
                                              userappserver_host,
                                              options.keyname,
                                              clear_datastore=False)

        app_exists = userappclient.does_app_exist(app_id)
        app_admin = userappclient.get_app_admin(app_id)
        if app_admin is not None and username != app_admin:
            raise AppScaleException("The given user doesn't own this application" + \
              ", so they can't upload an app with that application ID. Please " + \
              "change the application ID and try again.")

        if app_exists:
            AppScaleLogger.log(
                "Uploading new version of app {0}".format(app_id))
        else:
            AppScaleLogger.log(
                "Uploading initial version of app {0}".format(app_id))
            userappclient.reserve_app_id(username, app_id, app_language)

        remote_file_path = RemoteHelper.copy_app_to_host(
            file_location, options.keyname, options.verbose)

        acc.done_uploading(app_id, remote_file_path)
        acc.update([app_id])

        # now that we've told the AppController to start our app, find out what port
        # the app is running on and wait for it to start serving
        AppScaleLogger.log("Please wait for your app to start serving.")

        if app_exists:
            time.sleep(20)  # give the AppController time to restart the app

        serving_host, serving_port = userappclient.get_serving_info(
            app_id, options.keyname)
        RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
                                              options.verbose)
        AppScaleLogger.success(
            "Your app can be reached at the following URL: " +
            "http://{0}:{1}".format(serving_host, serving_port))

        if created_dir:
            shutil.rmtree(file_location)

        return (serving_host, serving_port)
Пример #53
0
  def run_instances(cls, options):
    """Starts a new AppScale deployment with the parameters given.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        When this occurs, the message in the exception contains the reason why
        the AppController crashed.
      BadConfigurationException: If the user passes in options that are not
        sufficient to start an AppScale deployment (e.g., running on EC2 but
        not specifying the AMI to use), or if the user provides us
        contradictory options (e.g., running on EC2 but not specifying EC2
        credentials).
    """
    LocalState.make_appscale_directory()
    LocalState.ensure_appscale_isnt_running(options.keyname, options.force)
    if options.infrastructure:
      if not options.disks and not options.test and not options.force:
        LocalState.ensure_user_wants_to_run_without_disks()

    reduced_version = '.'.join(x for x in APPSCALE_VERSION.split('.')[:2])
    AppScaleLogger.log("Starting AppScale " + reduced_version)

    my_id = str(uuid.uuid4())
    AppScaleLogger.remote_log_tools_state(options, my_id, "started",
      APPSCALE_VERSION)

    node_layout = NodeLayout(options)
    if not node_layout.is_valid():
      raise BadConfigurationException("There were errors with your " + \
                                      "placement strategy:\n{0}".format(str(node_layout.errors())))

    head_node = node_layout.head_node()
    # Start VMs in cloud via cloud agent.
    if options.infrastructure:
      instance_ids, public_ips, private_ips = RemoteHelper.start_all_nodes(
        options, len(node_layout.nodes))
      AppScaleLogger.log("\nPlease wait for AppScale to prepare your machines "
                         "for use. This can take few minutes.")

      # Set newly obtained node layout info for this deployment.
      for i, _ in enumerate(instance_ids):
        node_layout.nodes[i].public_ip = public_ips[i]
        node_layout.nodes[i].private_ip = private_ips[i]
        node_layout.nodes[i].instance_id = instance_ids[i]

      # Enables root logins and SSH access on the head node.
      RemoteHelper.enable_root_ssh(options, head_node.public_ip)
    AppScaleLogger.verbose("Node Layout: {}".format(node_layout.to_list()),
                           options.verbose)

    # Ensure all nodes are compatible.
    RemoteHelper.ensure_machine_is_compatible(
      head_node.public_ip, options.keyname, options.verbose)

    # Use rsync to move custom code into the deployment.
    if options.scp:
      AppScaleLogger.log("Copying over local copy of AppScale from {0}".
        format(options.scp))
      RemoteHelper.rsync_files(head_node.public_ip, options.keyname, options.scp,
        options.verbose)

    # Start services on head node.
    RemoteHelper.start_head_node(options, my_id, node_layout)

    # Write deployment metadata to disk (facilitates SSH operations, etc.)
    db_master = node_layout.db_master().private_ip
    head_node = node_layout.head_node().public_ip
    LocalState.update_local_metadata(options, db_master, head_node)

    # Copy the locations.json to the head node
    RemoteHelper.copy_local_metadata(node_layout.head_node().public_ip,
                                     options.keyname, options.verbose)

    # Wait for services on head node to start.
    secret_key = LocalState.get_secret_key(options.keyname)
    acc = AppControllerClient(head_node, secret_key)
    try:
      while not acc.is_initialized():
        AppScaleLogger.log('Waiting for head node to initialize...')
        # This can take some time in particular the first time around, since
        # we will have to initialize the database.
        time.sleep(cls.SLEEP_TIME*3)
    except socket.error as socket_error:
      AppScaleLogger.warn('Unable to initialize AppController: {}'.
                          format(socket_error.message))
      message = RemoteHelper.collect_appcontroller_crashlog(
        head_node, options.keyname, options.verbose)
      raise AppControllerException(message)

    # Set up admin account.
    try:
      # We don't need to have any exception information here: we do expect
      # some anyway while the UserAppServer is coming up.
      acc.does_user_exist("non-existent-user", True)
    except Exception:
      AppScaleLogger.log('UserAppServer not ready yet. Retrying ...')
      time.sleep(cls.SLEEP_TIME)

    if options.admin_user and options.admin_pass:
      AppScaleLogger.log("Using the provided admin username/password")
      username, password = options.admin_user, options.admin_pass
    elif options.test:
      AppScaleLogger.log("Using default admin username/password")
      username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
    else:
      username, password = LocalState.get_credentials()

    RemoteHelper.create_user_accounts(username, password, head_node,
                                      options.keyname)
    acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)

    # Wait for machines to finish loading and AppScale Dashboard to be deployed.
    RemoteHelper.wait_for_machines_to_finish_loading(head_node, options.keyname)
    RemoteHelper.sleep_until_port_is_open(LocalState.get_login_host(
      options.keyname), RemoteHelper.APP_DASHBOARD_PORT, options.verbose)

    AppScaleLogger.success("AppScale successfully started!")
    AppScaleLogger.success("View status information about your AppScale " + \
                           "deployment at http://{0}:{1}".format(LocalState.get_login_host(
                           options.keyname), RemoteHelper.APP_DASHBOARD_PORT))
    AppScaleLogger.remote_log_tools_state(options, my_id,
      "finished", APPSCALE_VERSION)
Пример #54
0
  def start_head_node(cls, options, my_id, node_layout):
    """Starts the first node in an AppScale deployment and instructs it to start
    API services on its own node, as well as the other nodes in the deployment.

    This includes spawning the first node in the deployment, copying over all
    deployment-specific files to it, and starting its AppController service.

    Args:
      options: A Namespace that includes parameters passed in by the user that
        define non-placement-strategy-related deployment options (e.g., keypair
        names, security group names).
      my_id: A str that is used to uniquely identify this AppScale deployment
        with the remote start application.
      node_layout: A NodeLayout that describes the placement strategy that
        should be used for this AppScale deployment.
    Returns:
      The public IP and instance ID (a dummy value in non-cloud deployments)
      corresponding to the node that was started.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        The message in this exception indicates why the crash occurred.
    """
    secret_key = LocalState.generate_secret_key(options.keyname)
    AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
      options.verbose)

    if options.infrastructure:
      instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(options)
    else:
      instance_id = cls.DUMMY_INSTANCE_ID
      public_ip = node_layout.head_node().public_ip
      private_ip = node_layout.head_node().private_ip

    AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
      LocalState.get_key_path_from_name(options.keyname), public_ip))

    try:
      cls.ensure_machine_is_compatible(public_ip, options.keyname,
        options.table, options.verbose)
    except AppScaleException as ase:
      # On failure shutdown the cloud instances, cleanup the keys, but only
      # if --test is not set.
      if options.infrastructure:
        if not options.test:
          try:
            cls.terminate_cloud_instance(instance_id, options)
          except Exception as tcie:
            AppScaleLogger.log("Error terminating instances: {0}"
              .format(str(tcie)))
        raise AppScaleException("{0} Please ensure that the "\
          "image {1} has AppScale {2} installed on it."
          .format(str(ase), options.machine, APPSCALE_VERSION))
      else:
        raise AppScaleException("{0} Please login to that machine and ensure "\
          "that AppScale {1} is installed on it."
          .format(str(ase), APPSCALE_VERSION))

    if options.scp:
      AppScaleLogger.log("Copying over local copy of AppScale from {0}".format(
        options.scp))
      cls.rsync_files(public_ip, options.keyname, options.scp, options.verbose)

    # On Euca, we've seen issues where attaching the EBS volume right after
    # the instance starts doesn't work. This sleep lets the instance fully
    # start up and get volumes attached to it correctly.
    if options.infrastructure and options.infrastructure == 'euca' and \
      options.disks:
      time.sleep(30)

    if options.infrastructure:
      agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
      params = agent.get_params_from_args(options)
      additional_params = {}

      if agent.PARAM_CREDENTIALS in params:
        additional_params = params[agent.PARAM_CREDENTIALS]

      if options.use_spot_instances:
        additional_params[agent.PARAM_SPOT_PRICE] = \
          str(params[agent.PARAM_SPOT_PRICE])

      if agent.PARAM_REGION in params:
        additional_params[agent.PARAM_REGION] = params[agent.PARAM_REGION]
    else:
      additional_params = {}

    deployment_params = LocalState.generate_deployment_params(options,
      node_layout, public_ip, additional_params)
    AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
      options.verbose)
    AppScaleLogger.log("Head node successfully initialized at {0}.".format(public_ip))

    AppScaleLogger.remote_log_tools_state(options, my_id, "started head node",
      APPSCALE_VERSION)
    time.sleep(10)  # gives machines in cloud extra time to boot up

    cls.copy_deployment_credentials(public_ip, options)
    cls.run_user_commands(public_ip, options.user_commands, options.keyname,
      options.verbose)
    cls.start_remote_appcontroller(public_ip, options.keyname, options.verbose)

    acc = AppControllerClient(public_ip, secret_key)
    locations = [{
      'public_ip' : public_ip,
      'private_ip' : private_ip,
      'jobs' : node_layout.head_node().roles,
      'instance_id' : instance_id,
      'disk' : node_layout.head_node().disk
    }]
    try:
      acc.set_parameters(locations, LocalState.map_to_array(deployment_params))
    except Exception as exception:
      AppScaleLogger.warn('Saw Exception while setting AC parameters: {0}' \
        .format(str(exception)))
      message = RemoteHelper.collect_appcontroller_crashlog(public_ip,
        options.keyname, options.verbose)
      raise AppControllerException(message)

    return public_ip, instance_id
Пример #55
0
  def start_head_node(cls, options, my_id, node_layout):
    """Starts the first node in an AppScale deployment and instructs it to start
    API services on its own node, as well as the other nodes in the deployment.

    This includes spawning the first node in the deployment, copying over all
    deployment-specific files to it, and starting its AppController service.

    Args:
      options: A Namespace that includes parameters passed in by the user that
        define non-placement-strategy-related deployment options (e.g., keypair
        names, security group names).
      my_id: A str that is used to uniquely identify this AppScale deployment
        with the remote start application.
      node_layout: A NodeLayout that describes the placement strategy that
        should be used for this AppScale deployment.
    Returns:
      The public IP and instance ID (a dummy value in non-cloud deployments)
      corresponding to the node that was started.
    """
    secret_key = LocalState.generate_secret_key(options.keyname)
    AppScaleLogger.verbose("Secret key is {0}".format(secret_key),
      options.verbose)

    if options.infrastructure:
      instance_id, public_ip, private_ip = cls.spawn_node_in_cloud(options)
    else:
      instance_id = cls.DUMMY_INSTANCE_ID
      public_ip = node_layout.head_node().public_ip
      private_ip = node_layout.head_node().private_ip

    AppScaleLogger.log("Log in to your head node: ssh -i {0} root@{1}".format(
      LocalState.get_key_path_from_name(options.keyname), public_ip))

    try:
      cls.ensure_machine_is_compatible(public_ip, options.keyname,
        options.table, options.verbose)
    except AppScaleException as ase:
       # On failure shutdown the cloud instances, cleanup the keys, but only 
       # if --test is not set.
       if options.infrastructure:
         if not options.test:
           try:
             cls.terminate_cloud_instance(instance_id, options)
           except Exception as tcie:
             AppScaleLogger.log("Error terminating instances: {0}"
               .format(str(tcie)))
         raise AppScaleException("{0} Please ensure that the "\
           "image {1} has AppScale {2} installed on it."
           .format(str(ase),options.machine,APPSCALE_VERSION))
       else:
         raise AppScaleException("{0} Please login to that machine and ensure "\
           "that AppScale {1} is installed on it."
           .format(str(ase),APPSCALE_VERSION))

    if options.scp:
      AppScaleLogger.log("Copying over local copy of AppScale from {0}".format(
        options.scp))
      cls.rsync_files(public_ip, options.keyname, options.scp, options.verbose)

    if options.infrastructure:
      agent = InfrastructureAgentFactory.create_agent(options.infrastructure)
      params = agent.get_params_from_args(options)
      additional_params = params[agent.PARAM_CREDENTIALS]

      if options.use_spot_instances:
        additional_params[agent.PARAM_SPOT_PRICE] = str(params[agent.PARAM_SPOT_PRICE])
    else:
      additional_params = {}

    deployment_params = LocalState.generate_deployment_params(options,
      node_layout, public_ip, additional_params)
    AppScaleLogger.verbose(str(LocalState.obscure_dict(deployment_params)),
      options.verbose)
    AppScaleLogger.log("Head node successfully initialized at {0}. It is now "\
      "starting up {1}.".format(public_ip, options.table))

    AppScaleLogger.remote_log_tools_state(options, my_id, "started head node",
      APPSCALE_VERSION)
    time.sleep(10)  # gives machines in cloud extra time to boot up

    cls.copy_deployment_credentials(public_ip, options)
    cls.start_remote_appcontroller(public_ip, options.keyname, options.verbose)

    acc = AppControllerClient(public_ip, secret_key)
    locations = ["{0}:{1}:{2}:{3}:cloud1".format(public_ip, private_ip,
      ":".join(node_layout.head_node().roles), instance_id)]
    acc.set_parameters(locations, LocalState.map_to_array(deployment_params))

    return public_ip, instance_id
Пример #56
0
    def run_instances(cls, options):
        """Starts a new AppScale deployment with the parameters given.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppControllerException: If the AppController on the head node crashes.
        When this occurs, the message in the exception contains the reason why
        the AppController crashed.
      BadConfigurationException: If the user passes in options that are not
        sufficient to start an AppScale deployment (e.g., running on EC2 but
        not specifying the AMI to use), or if the user provides us
        contradictory options (e.g., running on EC2 but not specifying EC2
        credentials).
    """
        LocalState.make_appscale_directory()
        LocalState.ensure_appscale_isnt_running(options.keyname, options.force)

        if options.infrastructure:
            if not options.disks and not options.test and not options.force:
                LocalState.ensure_user_wants_to_run_without_disks()
            AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
                               " over the " + options.infrastructure +
                               " cloud.")
        else:
            AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
                               " over a virtualized cluster.")
        my_id = str(uuid.uuid4())
        AppScaleLogger.remote_log_tools_state(options, my_id, "started",
                                              APPSCALE_VERSION)

        node_layout = NodeLayout(options)
        if not node_layout.is_valid():
            raise BadConfigurationException("There were errors with your " + \
              "placement strategy:\n{0}".format(str(node_layout.errors())))

        public_ip, instance_id = RemoteHelper.start_head_node(
            options, my_id, node_layout)
        AppScaleLogger.log(
            "\nPlease wait for AppScale to prepare your machines " +
            "for use. This can take few minutes.")

        # Write our metadata as soon as possible to let users SSH into those
        # machines via 'appscale ssh'.
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        acc = AppControllerClient(public_ip,
                                  LocalState.get_secret_key(options.keyname))

        # Let's now wait till the server is initialized.
        while not acc.is_initialized():
            AppScaleLogger.log('Waiting for head node to initialize...')
            # This can take some time in particular the first time around, since
            # we will have to initialize the database.
            time.sleep(cls.SLEEP_TIME * 3)

        try:
            # We don't need to have any exception information here: we do expect
            # some anyway while the UserAppServer is coming up.
            acc.does_user_exist("non-existent-user", True)
        except Exception as exception:
            AppScaleLogger.log('UserAppServer not ready yet. Retrying ...')
            time.sleep(cls.SLEEP_TIME)

        # Update our metadata again so that users can SSH into other boxes that
        # may have been started.
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        if options.admin_user and options.admin_pass:
            AppScaleLogger.log("Using the provided admin username/password")
            username, password = options.admin_user, options.admin_pass
        elif options.test:
            AppScaleLogger.log("Using default admin username/password")
            username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
        else:
            username, password = LocalState.get_credentials()

        RemoteHelper.create_user_accounts(username, password, public_ip,
                                          options.keyname,
                                          options.clear_datastore)
        acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)

        RemoteHelper.wait_for_machines_to_finish_loading(
            public_ip, options.keyname)
        # Finally, update our metadata once we know that all of the machines are
        # up and have started all their API services.
        LocalState.update_local_metadata(options, node_layout, public_ip,
                                         instance_id)
        RemoteHelper.copy_local_metadata(public_ip, options.keyname,
                                         options.verbose)

        RemoteHelper.sleep_until_port_is_open(
            LocalState.get_login_host(options.keyname),
            RemoteHelper.APP_DASHBOARD_PORT, options.verbose)
        AppScaleLogger.success("AppScale successfully started!")
        AppScaleLogger.success("View status information about your AppScale " + \
          "deployment at http://{0}:{1}/status".format(LocalState.get_login_host(
          options.keyname), RemoteHelper.APP_DASHBOARD_PORT))
        AppScaleLogger.remote_log_tools_state(options, my_id, "finished",
                                              APPSCALE_VERSION)
Пример #57
0
  def run_instances(cls, options):
    """Starts a new AppScale deployment with the parameters given.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      BadConfigurationException: If the user passes in options that are not
        sufficient to start an AppScale deplyoment (e.g., running on EC2 but
        not specifying the AMI to use), or if the user provides us
        contradictory options (e.g., running on EC2 but not specifying EC2
        credentials).
    """
    LocalState.make_appscale_directory()
    LocalState.ensure_appscale_isnt_running(options.keyname, options.force)

    if options.infrastructure:
      AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
        " over the " + options.infrastructure + " cloud.")
    else:
      AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
        " over a virtualized cluster.")
    my_id = str(uuid.uuid4())
    AppScaleLogger.remote_log_tools_state(options, my_id, "started",
      APPSCALE_VERSION)

    node_layout = NodeLayout(options)
    if not node_layout.is_valid():
      raise BadConfigurationException("There were errors with your " + \
        "placement strategy:\n{0}".format(str(node_layout.errors())))

    if not node_layout.is_supported():
      AppScaleLogger.warn("Warning: This deployment strategy is not " + \
        "officially supported.")

    public_ip, instance_id = RemoteHelper.start_head_node(options, my_id,
      node_layout)
    AppScaleLogger.log("\nPlease wait for AppScale to prepare your machines " +
      "for use.")

    # Write our metadata as soon as possible to let users SSH into those
    # machines via 'appscale ssh'
    LocalState.update_local_metadata(options, node_layout, public_ip,
      instance_id)
    RemoteHelper.copy_local_metadata(public_ip, options.keyname,
      options.verbose)

    acc = AppControllerClient(public_ip, LocalState.get_secret_key(
      options.keyname))
    uaserver_host = acc.get_uaserver_host(options.verbose)

    RemoteHelper.sleep_until_port_is_open(uaserver_host, UserAppClient.PORT,
      options.verbose)

    # Update our metadata again so that users can SSH into other boxes that
    # may have been started.
    LocalState.update_local_metadata(options, node_layout, public_ip,
      instance_id)
    RemoteHelper.copy_local_metadata(public_ip, options.keyname,
      options.verbose)

    AppScaleLogger.log("UserAppServer is at {0}".format(uaserver_host))

    uaserver_client = UserAppClient(uaserver_host,
      LocalState.get_secret_key(options.keyname))

    if options.admin_user and options.admin_pass:
      AppScaleLogger.log("Using the provided admin username/password")
      username, password = options.admin_user, options.admin_pass
    elif options.test:
      AppScaleLogger.log("Using default admin username/password")
      username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
    else:
      username, password = LocalState.get_credentials()

    RemoteHelper.create_user_accounts(username, password, uaserver_host,
      options.keyname)
    uaserver_client.set_admin_role(username)

    RemoteHelper.wait_for_machines_to_finish_loading(public_ip, options.keyname)
    # Finally, update our metadata once we know that all of the machines are
    # up and have started all their API services.
    LocalState.update_local_metadata(options, node_layout, public_ip,
      instance_id)
    RemoteHelper.copy_local_metadata(public_ip, options.keyname, options.verbose)

    RemoteHelper.sleep_until_port_is_open(LocalState.get_login_host(
      options.keyname), RemoteHelper.APP_LOAD_BALANCER_PORT, options.verbose)
    AppScaleLogger.success("AppScale successfully started!")
    AppScaleLogger.success("View status information about your AppScale " + \
      "deployment at http://{0}/status".format(LocalState.get_login_host(
      options.keyname)))
    AppScaleLogger.remote_log_tools_state(options, my_id,
      "finished", APPSCALE_VERSION)
Пример #58
0
    def upload_app(cls, options):
        """Uploads the given App Engine application into AppScale.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Returns:
      A tuple containing the host and port where the application is serving
        traffic from.
    """
        if cls.TAR_GZ_REGEX.search(options.file):
            file_location = LocalState.extract_tgz_app_to_dir(
                options.file, options.verbose)
            created_dir = True
        elif cls.ZIP_REGEX.search(options.file):
            file_location = LocalState.extract_zip_app_to_dir(
                options.file, options.verbose)
            created_dir = True
        elif os.path.isdir(options.file):
            file_location = options.file
            created_dir = False
        else:
            raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
              'or a directory. Please try uploading either a tar.gz file, a zip ' \
              'file, or a directory.'.format(options.file))

        try:
            app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
        except AppEngineConfigException as config_error:
            AppScaleLogger.log(config_error)
            if 'yaml' in str(config_error):
                raise config_error

            # Java App Engine users may have specified their war directory. In that
            # case, just move up one level, back to the app's directory.
            file_location = file_location + os.sep + ".."
            app_id = AppEngineHelper.get_app_id_from_app_config(file_location)

        app_language = AppEngineHelper.get_app_runtime_from_app_config(
            file_location)
        AppEngineHelper.validate_app_id(app_id)

        if app_language == 'java':
            if AppEngineHelper.is_sdk_mismatch(file_location):
                AppScaleLogger.warn(
                    'AppScale did not find the correct SDK jar ' +
                    'versions in your app. The current supported ' +
                    'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION +
                    '.')

        login_host = LocalState.get_login_host(options.keyname)
        secret_key = LocalState.get_secret_key(options.keyname)
        acc = AppControllerClient(login_host, secret_key)

        if options.test:
            username = LocalState.DEFAULT_USER
        elif options.email:
            username = options.email
        else:
            username = LocalState.get_username_from_stdin(is_admin=False)

        if not acc.does_user_exist(username):
            password = LocalState.get_password_from_stdin()
            RemoteHelper.create_user_accounts(username,
                                              password,
                                              login_host,
                                              options.keyname,
                                              clear_datastore=False)

        app_exists = acc.does_app_exist(app_id)
        app_admin = acc.get_app_admin(app_id)
        if app_admin is not None and username != app_admin:
            raise AppScaleException("The given user doesn't own this application" + \
              ", so they can't upload an app with that application ID. Please " + \
              "change the application ID and try again.")

        if app_exists:
            AppScaleLogger.log(
                "Uploading new version of app {0}".format(app_id))
        else:
            AppScaleLogger.log(
                "Uploading initial version of app {0}".format(app_id))
            acc.reserve_app_id(username, app_id, app_language)

        # Ignore all .pyc files while tarring.
        if app_language == 'python27':
            AppScaleLogger.log("Ignoring .pyc files")

        remote_file_path = RemoteHelper.copy_app_to_host(
            file_location, options.keyname, options.verbose)

        acc.done_uploading(app_id, remote_file_path)
        acc.update([app_id])

        # now that we've told the AppController to start our app, find out what port
        # the app is running on and wait for it to start serving
        AppScaleLogger.log("Please wait for your app to start serving.")

        if app_exists:
            time.sleep(20)  # give the AppController time to restart the app

        # Makes a call to the AppController to get all the stats and looks
        # through them for the http port the app can be reached on.
        sleep_time = 2 * cls.SLEEP_TIME
        current_app = None
        for i in range(cls.MAX_RETRIES):
            try:
                result = acc.get_all_stats()
                json_result = json.loads(result)
                apps_result = json_result['apps']
                current_app = apps_result[app_id]
                http_port = current_app['http']
                break
            except ValueError:
                pass
            except KeyError:
                pass
            AppScaleLogger.verbose("Waiting {0} second(s) for a port to be assigned to {1}".\
              format(sleep_time, app_id), options.verbose)
            time.sleep(sleep_time)
        if not current_app:
            raise AppScaleException(
                "Unable to get the serving port for the application.")

        RemoteHelper.sleep_until_port_is_open(login_host, http_port,
                                              options.verbose)
        AppScaleLogger.success(
            "Your app can be reached at the following URL: " +
            "http://{0}:{1}".format(login_host, http_port))

        if created_dir:
            shutil.rmtree(file_location)

        return (login_host, http_port)
  def test_appscale_in_one_node_virt_deployment(self):
    self.local_state.should_receive('shell').with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin="cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/")

    self.local_state.should_receive('shell').with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] ", False, 5, stdin="chmod +x /etc/init.d/appcontroller")
    
    self.local_state.should_receive('shell').with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin="cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/")

    # let's say that appscale isn't already running
    self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
    self.local_state.should_receive('make_appscale_directory').and_return()

    rh = flexmock(RemoteHelper)
    rh.should_receive('copy_deployment_credentials').and_return()

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    # mock out copying over the keys
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.key'),False,5)

    self.setup_appscale_compatibility_mocks()

    # mock out generating the private key
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^openssl'),False,stdin=None)\
      .and_return()

    # mock out removing the old json file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
      .and_return()

    # assume that we started monit fine
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
      .and_return()

    # and that we copied over the AppController's monit file
    self.local_state.should_receive('shell')\
      .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
      .and_return()

    self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() 

    self.setup_socket_mocks('1.2.3.4')
    self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')

    # mock out reading the locations.json file, and slip in our own json
    self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
      json.dumps([{
        "public_ip" : "1.2.3.4",
        "private_ip" : "1.2.3.4",
        "jobs" : ["shadow", "login"]
      }])))

    # Assume the locations files were copied successfully.
    locations_file = '{}/locations-bookey.yaml'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
      .and_return()

    locations_json = '{}/locations-bookey.json'.\
      format(RemoteHelper.CONFIG_DIR)
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
      .and_return()

    user_locations = '/root/.appscale/locations-bookey.json'
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
      .and_return()

    # Assume the secret key was copied successfully.
    self.local_state.should_receive('shell')\
      .with_args(re.compile('^scp .*.secret'), False, 5)\
      .and_return()

    flexmock(AppControllerClient)
    AppControllerClient.should_receive('does_user_exist').and_return(True)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine:  1.2.3.4
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test"
    ]

    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)