Пример #1
0
    def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
        """Copies over an AppScale source directory from this machine to the
    specified host.

    Args:
      host: A str representing a host that should be accessible from this
        machine.
      keyname: A str representing the name of the SSH keypair that can log into
        the specified machine.
      local_appscale_dir: A str representing the path on the local filesystem
        where the AppScale source to copy over can be found.
      is_verbose: A bool that indicates if we should print the rsync commands
        we exec to stdout.
    Raises:
      BadConfigurationException: If local_appscale_dir does not exist locally,
        or if any of the standard AppScale module folders do not exist.
    """
        ssh_key = LocalState.get_key_path_from_name(keyname)
        local_path = os.path.expanduser(local_appscale_dir)
        if not os.path.exists(local_path):
            raise BadConfigurationException("The location you specified to copy " \
              "from, {0}, doesn't exist.".format(local_path))
        LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
          "--exclude='AppDB/logs/*' " \
          "--exclude='AppDB/cassandra/cassandra/*' " \
          "{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
          local_path, host), is_verbose)
Пример #2
0
    def describe_instances(cls, options):
        """Queries each node in the currently running AppScale deployment and
    reports on their status.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        login_host = LocalState.get_login_host(options.keyname)
        login_acc = AppControllerClient(
            login_host, LocalState.get_secret_key(options.keyname))

        for ip in login_acc.get_all_public_ips():
            acc = AppControllerClient(
                ip, LocalState.get_secret_key(options.keyname))
            AppScaleLogger.log("Status of node at {0}:".format(ip))
            try:
                AppScaleLogger.log(acc.get_status())
            except Exception as exception:
                AppScaleLogger.warn("Unable to contact machine: {0}\n".format(
                    str(exception)))

        AppScaleLogger.success("View status information about your AppScale " + \
          "deployment at http://{0}:{1}/status".format(login_host,
          RemoteHelper.APP_DASHBOARD_PORT))
Пример #3
0
    def terminate_virtualized_cluster(cls, keyname, is_verbose):
        """Stops all API services running on all nodes in the currently running
    AppScale deployment.

    Args:
      keyname: The name of the SSH keypair used for this AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands executed
        to stdout.
    """
        AppScaleLogger.log(
            "Terminating appscale deployment with keyname {0}".format(keyname))
        time.sleep(2)

        shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
        try:
            secret = LocalState.get_secret_key(keyname)
        except IOError:
            # We couldn't find the secret key: AppScale is most likely not
            # running.
            raise AppScaleException("Couldn't find AppScale secret key.")

        acc = AppControllerClient(shadow_host, secret)
        try:
            all_ips = acc.get_all_public_ips()
        except Exception as exception:
            AppScaleLogger.warn(
                'Saw Exception while getting deployments IPs {0}'.format(
                    str(exception)))
            all_ips = LocalState.get_all_public_ips(keyname)

        threads = []
        for ip in all_ips:
            thread = threading.Thread(target=cls.stop_remote_appcontroller,
                                      args=(ip, keyname, is_verbose))
            thread.start()
            threads.append(thread)

        for thread in threads:
            thread.join()

        boxes_shut_down = 0
        is_running_regex = re.compile("appscale-controller stop")
        for ip in all_ips:
            AppScaleLogger.log(
                "Shutting down AppScale API services at {0}".format(ip))
            while True:
                remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
                AppScaleLogger.verbose(remote_output, is_verbose)
                if not is_running_regex.match(remote_output):
                    break
                time.sleep(0.3)
            boxes_shut_down += 1

        if boxes_shut_down != len(all_ips):
            raise AppScaleException(
                "Couldn't terminate your AppScale deployment on"
                " all machines - please do so manually.")

        AppScaleLogger.log(
            "Terminated AppScale on {0} machines.".format(boxes_shut_down))
Пример #4
0
    def remove_app(cls, options):
        """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if not options.confirm:
            response = raw_input("Are you sure you want to remove this " + \
              "application? (Y/N) ")
            if response not in ['y', 'yes', 'Y', 'YES']:
                raise AppScaleException("Cancelled application removal.")

        login_host = LocalState.get_login_host(options.keyname)
        secret = LocalState.get_secret_key(options.keyname)
        acc = AppControllerClient(login_host, secret)

        if not acc.does_app_exist(options.appname):
            raise AppScaleException(
                "The given application is not currently running.")

        acc.stop_app(options.appname)
        AppScaleLogger.log("Please wait for your app to shut down.")
        while True:
            if acc.is_app_running(options.appname):
                time.sleep(cls.SLEEP_TIME)
            else:
                break
        AppScaleLogger.success("Done shutting down {0}".format(
            options.appname))
Пример #5
0
    def copy_app_to_host(cls, app_location, keyname, is_verbose):
        """Copies the given application to a machine running the Login service
    within an AppScale deployment.

    Args:
      app_location: The location on the local filesystem where the application
        can be found.
      keyname: The name of the SSH keypair that uniquely identifies this
        AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands we exec
        to copy the app to the remote host to stdout.

    Returns:
      A str corresponding to the location on the remote filesystem where the
        application was copied to.
    """
        app_id = AppEngineHelper.get_app_id_from_app_config(app_location)

        AppScaleLogger.log("Tarring application")
        rand = str(uuid.uuid4()).replace('-', '')[:8]
        local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(
            tempfile.gettempdir(), app_id, rand)
        LocalState.shell(
            "cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".
            format(app_location, local_tarred_app), is_verbose)

        AppScaleLogger.log("Copying over application")
        remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
        cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
                remote_app_tar, is_verbose)

        os.remove(local_tarred_app)
        return remote_app_tar
Пример #6
0
    def terminate_instances(cls, options):
        """Stops all services running in an AppScale deployment, and in cloud
    deployments, also powers off the instances previously spawned.

    Raises:
      AppScaleException: If AppScale is not running, and thus can't be
      terminated.
    """
        if not os.path.exists(
                LocalState.get_locations_yaml_location(options.keyname)):
            raise AppScaleException(
                "AppScale is not running with the keyname {0}".format(
                    options.keyname))

        if LocalState.get_infrastructure(options.keyname) in \
          InfrastructureAgentFactory.VALID_AGENTS:
            RemoteHelper.terminate_cloud_infrastructure(
                options.keyname, options.verbose)
        else:
            RemoteHelper.terminate_virtualized_cluster(options.keyname,
                                                       options.verbose)

        LocalState.cleanup_appscale_files(options.keyname)
        AppScaleLogger.success(
            "Successfully shut down your AppScale deployment.")
Пример #7
0
  def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
    """Copies over an AppScale source directory from this machine to the
    specified host.

    Args:
      host: A str representing a host that should be accessible from this
        machine.
      keyname: A str representing the name of the SSH keypair that can log into
        the specified machine.
      local_appscale_dir: A str representing the path on the local filesystem
        where the AppScale source to copy over can be found.
      is_verbose: A bool that indicates if we should print the rsync commands
        we exec to stdout.
    Raises:
      BadConfigurationException: If local_appscale_dir does not exist locally,
        or if any of the standard AppScale module folders do not exist.
    """
    ssh_key = LocalState.get_key_path_from_name(keyname)
    local_path = os.path.expanduser(local_appscale_dir)
    if not os.path.exists(local_path):
      raise BadConfigurationException("The location you specified to copy " \
        "from, {0}, doesn't exist.".format(local_path))
    LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
      "--exclude='AppDB/logs/*' " \
      "--exclude='AppDB/cassandra/cassandra/*' " \
      "{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
      local_path, host), is_verbose)
Пример #8
0
    def scp_remote_to_local(cls,
                            host,
                            keyname,
                            source,
                            dest,
                            is_verbose,
                            user='******'):
        """Securely copies a file from a remote machine to this machine.

    Args:
      host: A str representing the machine that we should log into.
      keyname: A str representing the name of the SSH keypair to log in with.
      source: A str representing the path on the remote machine where the
        file should be copied from.
      dest: A str representing the path on the local machine where the file
        should be copied to.
      is_verbose: A bool that indicates if we should print the scp command to
        stdout.
      user: A str representing the user to log in as.
    Returns:
      A str representing the standard output of the secure copy and a str
        representing the standard error of the secure copy.
    """
        ssh_key = LocalState.get_key_path_from_name(keyname)
        return LocalState.shell(
            "scp -r -i {0} {1} {2}@{3}:{4} {5}".format(ssh_key,
                                                       cls.SSH_OPTIONS, user,
                                                       host, source, dest),
            is_verbose)
Пример #9
0
  def testCleanInClusterDeployment(self):
    # calling 'appscale clean' in a cluster deployment should ssh into each of
    # the boxes specified in the ips_layout and run the terminate script

    # Mock out the actual file reading itself, and slip in a YAML-dumped
    # file
    contents = {
      'ips_layout' : {
        'controller': 'public1',
        'servers': ['public2', 'public3']
      },
      'test' : True
    }
    yaml_dumped_contents = yaml.dump(contents)

    flexmock(RemoteHelper)
    RemoteHelper.should_receive('ssh') \
      .with_args(re.compile('public[123]'), 'appscale', str, False)

    flexmock(LocalState)
    LocalState.should_receive('cleanup_appscale_files').with_args('appscale')

    appscale = AppScale()
    self.addMockForAppScalefile(appscale, yaml_dumped_contents)
    expected = ['public1', 'public2', 'public3']
    self.assertEquals(expected, appscale.clean())
Пример #10
0
    def test_appscale_in_two_node_virt_deployment(self):
        # pretend that the place we're going to put logs into doesn't exist
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
            False)

        # and mock out the mkdir operation
        flexmock(os)
        os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

        # next, mock out finding the login ip address
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_nodes_json")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([{
                "public_ip": "public1",
                "private_ip": "private1",
                "jobs": ["shadow", "login"]
            }]))
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # and slip in a fake appcontroller to report on the two IP addrs
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('get_all_public_ips').with_args(
            'the secret').and_return(json.dumps(['public1', 'public2']))
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        # fake the creation of the log directories locally
        os.should_receive('mkdir').with_args(
            '/tmp/foobaz/public1').and_return()
        os.should_receive('mkdir').with_args(
            '/tmp/foobaz/public2').and_return()

        # finally, fake the copying of the log files
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.gather_logs(options)
Пример #11
0
  def test_ensure_appscale_isnt_running_but_it_is_w_force(self):
    # if there is a locations.yaml file and force is set,
    # we shouldn't abort
    os.path.should_receive('exists').with_args(self.locations_yaml) \
      .and_return(True)

    LocalState.ensure_appscale_isnt_running(self.keyname, True)
Пример #12
0
    def gather_logs(cls, options):
        """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        # First, make sure that the place we want to store logs doesn't
        # already exist.
        if os.path.exists(options.location):
            raise AppScaleException("Can't gather logs, as the location you " + \
              "specified, {0}, already exists.".format(options.location))

        acc = AppControllerClient(LocalState.get_login_host(options.keyname),
                                  LocalState.get_secret_key(options.keyname))

        # do the mkdir after we get the secret key, so that a bad keyname will
        # cause the tool to crash and not create this directory
        os.mkdir(options.location)

        for ip in acc.get_all_public_ips():
            # Get the logs from each node, and store them in our local directory
            local_dir = "{0}/{1}".format(options.location, ip)
            os.mkdir(local_dir)
            RemoteHelper.scp_remote_to_local(ip, options.keyname,
                                             '/var/log/appscale', local_dir,
                                             options.verbose)
        AppScaleLogger.success("Successfully copied logs to {0}".format(
            options.location))
Пример #13
0
    def ssh(cls,
            host,
            keyname,
            command,
            is_verbose,
            user='******',
            num_retries=LocalState.DEFAULT_NUM_RETRIES):
        """Logs into the named host and executes the given command.

    Args:
      host: A str representing the machine that we should log into.
      keyname: A str representing the name of the SSH keypair to log in with.
      command: A str representing what to execute on the remote host.
      is_verbose: A bool indicating if we should print the ssh command to
        stdout.
      user: A str representing the user to log in as.
    Returns:
      A str representing the standard output of the remote command and a str
        representing the standard error of the remote command.
    """
        ssh_key = LocalState.get_key_path_from_name(keyname)
        return LocalState.shell(
            "ssh -F /dev/null -i {0} {1} {2}@{3} bash".format(
                ssh_key, cls.SSH_OPTIONS, user, host),
            is_verbose,
            num_retries,
            stdin=command)
Пример #14
0
    def terminate_instances(cls, options):
        """Stops all services running in an AppScale deployment, and in cloud
    deployments, also powers off the instances previously spawned.

    Raises:
      AppScaleException: If AppScale is not running, and thus can't be
      terminated.
    """
        try:
            infrastructure = LocalState.get_infrastructure(options.keyname)
        except IOError:
            raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".format(options.keyname))

        if infrastructure == "xen" and options.terminate:
            raise AppScaleException("Terminate option is invalid for cluster mode.")

        if infrastructure == "xen" or not options.terminate:
            # We are in cluster mode: let's check if AppScale is running.
            if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
                raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))

        # Stop gracefully the AppScale deployment.
        try:
            RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)
        except (IOError, AppScaleException):
            # Don't fail if we cannot find the configuration.
            pass

        # And if we are on a cloud infrastructure, terminate instances if
        # asked.
        if infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate:
            RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
Пример #15
0
  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(options.location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {0}, already exists.".format(options.location))

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(options.location)

    for ip in acc.get_all_public_ips():
      # Get the logs from each node, and store them in our local directory
      local_dir = "{0}/{1}".format(options.location, ip)
      os.mkdir(local_dir)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
        local_dir, options.verbose)
    AppScaleLogger.success("Successfully copied logs to {0}".format(
      options.location))
Пример #16
0
  def add_instances(cls, options):
    """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if 'master' in options.ips.keys():
      raise BadConfigurationException("Cannot add master nodes to an " + \
        "already running AppScale deployment.")

    # Skip checking for -n (replication) because we don't allow the user
    # to specify it here (only allowed in run-instances).
    additional_nodes_layout = NodeLayout(options)

    # In virtualized cluster deployments, we need to make sure that the user
    # has already set up SSH keys.
    if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
      for ip in options.ips.values():
        # throws a ShellException if the SSH key doesn't work
        RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)

    # Finally, find an AppController and send it a message to add
    # the given nodes with the new roles.
    AppScaleLogger.log("Sending request to add instances")
    login_ip = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_ip, LocalState.get_secret_key(
      options.keyname))
    acc.start_roles_on_nodes(json.dumps(options.ips))

    # TODO(cgb): Should we wait for the new instances to come up and get
    # initialized?
    AppScaleLogger.success("Successfully sent request to add instances " + \
      "to this AppScale deployment.")
Пример #17
0
  def remove_app(cls, options):
    """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if not options.confirm:
      response = raw_input("Are you sure you want to remove this " + \
        "application? (Y/N) ")
      if response not in ['y', 'yes', 'Y', 'YES']:
        raise AppScaleException("Cancelled application removal.")

    login_host = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_host, LocalState.get_secret_key(
      options.keyname))
    userappserver_host = acc.get_uaserver_host(options.verbose)
    userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
      options.keyname))
    if not userappclient.does_app_exist(options.appname):
      raise AppScaleException("The given application is not currently running.")

    acc.stop_app(options.appname)
    AppScaleLogger.log("Please wait for your app to shut down.")
    while True:
      if acc.is_app_running(options.appname):
        time.sleep(cls.SLEEP_TIME)
      else:
        break
    AppScaleLogger.success("Done shutting down {0}".format(options.appname))
Пример #18
0
  def relocate_app(cls, options):
    """Instructs AppScale to move the named application to a different port.

    Args:
      options: A Namespace that has fields for each parameter that can be passed
        in via the command-line interface.
    Raises:
      AppScaleException: If the named application isn't running in this AppScale
        cloud, if the destination port is in use by a different application, or
        if the AppController rejects the request to relocate the application (in
        which case it includes the reason why the rejection occurred).
    """
    login_host = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_host, LocalState.get_secret_key(
      options.keyname))

    app_info_map = acc.get_app_info_map()
    if options.appname not in app_info_map.keys():
      raise AppScaleException("The given application, {0}, is not currently " \
        "running in this AppScale cloud, so we can't move it to a different " \
        "port.".format(options.appname))

    relocate_result = acc.relocate_app(options.appname, options.http_port,
      options.https_port)
    if relocate_result == "OK":
      AppScaleLogger.success("Successfully issued request to move {0} to " \
        "ports {1} and {2}.".format(options.appname, options.http_port,
        options.https_port))
      AppScaleLogger.success("Your app serves unencrypted traffic at: " +
        "http://{0}:{1}".format(login_host, options.http_port))
      AppScaleLogger.success("Your app serves encrypted traffic at: " +
        "https://{0}:{1}".format(login_host, options.https_port))
    else:
      raise AppScaleException(relocate_result)
Пример #19
0
  def copy_app_to_host(cls, app_location, keyname, is_verbose):
    """Copies the given application to a machine running the Login service
    within an AppScale deployment.

    Args:
      app_location: The location on the local filesystem where the application
        can be found.
      keyname: The name of the SSH keypair that uniquely identifies this
        AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands we exec
        to copy the app to the remote host to stdout.

    Returns:
      A str corresponding to the location on the remote filesystem where the
        application was copied to.
    """
    app_id = AppEngineHelper.get_app_id_from_app_config(app_location)

    AppScaleLogger.log("Tarring application")
    rand = str(uuid.uuid4()).replace('-', '')[:8]
    local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(tempfile.gettempdir(),
      app_id, rand)
    LocalState.shell("cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".format(
      app_location, local_tarred_app), is_verbose)

    AppScaleLogger.log("Copying over application")
    remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
    cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
      remote_app_tar, is_verbose)

    os.remove(local_tarred_app)
    return remote_app_tar
Пример #20
0
    def copy_local_metadata(cls, host, keyname, is_verbose):
        """Copies the locations.yaml and locations.json files found locally (which
    contain metadata about this AppScale deployment) to the specified host.

    Args:
      host: The machine that we should copy the metadata files to.
      keyname: The name of the SSH keypair that we can use to log into the given
        host.
      is_verbose: A bool that indicates if we should print the SCP commands we
        exec to stdout.
    """
        # copy the metadata files for AppScale itself to use
        cls.scp(host, keyname, LocalState.get_locations_yaml_location(keyname),
                '/etc/appscale/locations-{0}.yaml'.format(keyname), is_verbose)
        cls.scp(host, keyname, LocalState.get_locations_json_location(keyname),
                '/etc/appscale/locations-{0}.json'.format(keyname), is_verbose)

        # and copy the json file if the tools on that box wants to use it
        cls.scp(host, keyname, LocalState.get_locations_json_location(keyname),
                '/root/.appscale/locations-{0}.json'.format(keyname),
                is_verbose)

        # and copy the secret file if the tools on that box wants to use it
        cls.scp(host, keyname, LocalState.get_secret_key_location(keyname),
                '/root/.appscale/', is_verbose)
Пример #21
0
  def get_params_from_yaml(self, keyname):
    """Searches through the locations.yaml file to build a dict containing the
    parameters necessary to interact with Amazon EC2.

    Args:
      keyname: The name of the SSH keypair that uniquely identifies this
        AppScale deployment.
    """
    params = {
      self.PARAM_CREDENTIALS : {},
      self.PARAM_GROUP : LocalState.get_group(keyname),
      self.PARAM_KEYNAME : keyname
    }

    zone = LocalState.get_zone(keyname)
    if zone:
      params[self.PARAM_REGION] = zone[:-1]
    else:
      params[self.PARAM_REGION] = self.DEFAULT_REGION


    for credential in self.REQUIRED_CREDENTIALS:
      if os.environ.get(credential):
        params[self.PARAM_CREDENTIALS][credential] = os.environ[credential]
      else:
        raise AgentConfigurationException("no " + credential)

    return params
    def test_ensure_appscale_isnt_running_but_it_is_w_force(self):
        # if there is a locations.yaml file and force is set,
        # we shouldn't abort
        os.path.should_receive('exists').with_args(self.locations_yaml) \
          .and_return(True)

        LocalState.ensure_appscale_isnt_running(self.keyname, True)
Пример #23
0
  def testCleanInClusterDeployment(self):
    # calling 'appscale clean' in a cluster deployment should ssh into each of
    # the boxes specified in the ips_layout and run the terminate script

    # Mock out the actual file reading itself, and slip in a YAML-dumped
    # file
    contents = {
      'ips_layout' : {
        'controller': 'public1',
        'servers': ['public2', 'public3']
      },
      'test' : True
    }
    yaml_dumped_contents = yaml.dump(contents)

    flexmock(RemoteHelper)
    RemoteHelper.should_receive('ssh') \
      .with_args(re.compile('public[123]'), 'appscale', str, False)

    flexmock(LocalState)
    LocalState.should_receive('cleanup_appscale_files').with_args('appscale')

    appscale = AppScale()
    self.addMockForAppScalefile(appscale, yaml_dumped_contents)
    expected = ['public1', 'public2', 'public3']
    self.assertEquals(expected, appscale.clean())
Пример #24
0
  def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
    """Copies over an AppScale source directory from this machine to the
    specified host.

    Args:
      host: A str representing a host that should be accessible from this
        machine.
      keyname: A str representing the name of the SSH keypair that can log into
        the specified machine.
      local_appscale_dir: A str representing the path on the local filesystem
        where the AppScale source to copy over can be found.
      is_verbose: A bool that indicates if we should print the rsync commands
        we exec to stdout.
    Raises:
      BadConfigurationException: If local_appscale_dir does not exist locally,
        or if any of the standard AppScale module folders do not exist.
    """
    ssh_key = LocalState.get_key_path_from_name(keyname)
    appscale_dirs = ["lib", "AppController", "AppManager", "AppServer",
      "AppLoadBalancer", "AppMonitoring", "Neptune", "InfrastructureManager"]
    for dir_name in appscale_dirs:
      local_path = os.path.expanduser(local_appscale_dir) + os.sep + dir_name
      if not os.path.exists(local_path):
        raise BadConfigurationException("The location you specified to copy " +
          "from, {0}, doesn't contain a {1} folder.".format(local_appscale_dir,
          local_path))
      LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv {2}/* root@{3}:/root/appscale/{4}" \
        .format(ssh_key, cls.SSH_OPTIONS, local_path, host, dir_name), is_verbose)

    # Rsync AppDB separately, as it has a lot of paths we may need to exclude
    # (e.g., built database binaries).
    local_app_db = os.path.expanduser(local_appscale_dir) + os.sep + "AppDB/*"
    LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv --exclude='logs/*' --exclude='hadoop-*' --exclude='hbase/hbase-*' --exclude='voldemort/voldemort/*' --exclude='cassandra/cassandra/*' {2} root@{3}:/root/appscale/AppDB".format(ssh_key, cls.SSH_OPTIONS, local_app_db, host), is_verbose)
Пример #25
0
  def create_user_accounts(cls, email, password, uaserver_host, keyname):
    """Registers two new user accounts with the UserAppServer.

    One account is the standard account that users log in with (via their
    e-mail address. The other is their XMPP account, so that they can log into
    any jabber-compatible service and send XMPP messages to their application
    (and receive them).

    Args:
      email: The e-mail address that should be registered for the user's
        standard account.
      password: The password that should be used for both the standard and XMPP
        accounts.
      uaserver_host: The location of a UserAppClient, that can create new user
        accounts.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
    uaserver = UserAppClient(uaserver_host, LocalState.get_secret_key(keyname))

    # first, create the standard account
    encrypted_pass = LocalState.encrypt_password(email, password)
    uaserver.create_user(email, encrypted_pass)

    # next, create the XMPP account. if the user's e-mail is [email protected], then that
    # means their XMPP account name is a@login_ip
    username_regex = re.compile('\A(.*)@')
    username = username_regex.match(email).groups()[0]
    xmpp_user = "******".format(username, LocalState.get_login_host(keyname))
    xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
    uaserver.create_user(xmpp_user, xmpp_pass)
    AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
Пример #26
0
    def add_instances(cls, options):
        """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        if 'master' in options.ips.keys():
            raise BadConfigurationException("Cannot add master nodes to an " + \
              "already running AppScale deployment.")

        # Skip checking for -n (replication) because we don't allow the user
        # to specify it here (only allowed in run-instances).
        additional_nodes_layout = NodeLayout(options)

        # In virtualized cluster deployments, we need to make sure that the user
        # has already set up SSH keys.
        if LocalState.get_from_yaml(options.keyname,
                                    'infrastructure') == "xen":
            for ip in options.ips.values():
                # throws a ShellException if the SSH key doesn't work
                RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)

        # Finally, find an AppController and send it a message to add
        # the given nodes with the new roles.
        AppScaleLogger.log("Sending request to add instances")
        login_ip = LocalState.get_login_host(options.keyname)
        acc = AppControllerClient(login_ip,
                                  LocalState.get_secret_key(options.keyname))
        acc.start_roles_on_nodes(json.dumps(options.ips))

        # TODO(cgb): Should we wait for the new instances to come up and get
        # initialized?
        AppScaleLogger.success("Successfully sent request to add instances " + \
          "to this AppScale deployment.")
Пример #27
0
  def copy_app_to_host(cls, app_location, keyname, is_verbose):
    """Copies the given application to a machine running the Login service within
    an AppScale deployment.

    Args:
      app_location: The location on the local filesystem where the application
        can be found.
      keyname: The name of the SSH keypair that uniquely identifies this
        AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands we exec
        to copy the app to the remote host to stdout.

    Returns:
      A str corresponding to the location on the remote filesystem where the
        application was copied to.
    """
    AppScaleLogger.log("Creating remote directory to copy app into")
    app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
    remote_app_dir = "/var/apps/{0}/app".format(app_id)
    cls.ssh(LocalState.get_login_host(keyname), keyname,
      'mkdir -p {0}'.format(remote_app_dir), is_verbose)

    AppScaleLogger.log("Tarring application")
    rand = str(uuid.uuid4()).replace('-', '')[:8]
    local_tarred_app = "/tmp/appscale-app-{0}-{1}.tar.gz".format(app_id, rand)
    LocalState.shell("cd {0} && tar -czf {1} *".format(app_location,
      local_tarred_app), is_verbose)

    AppScaleLogger.log("Copying over application")
    remote_app_tar = "{0}/{1}.tar.gz".format(remote_app_dir, app_id)
    cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
      remote_app_tar, is_verbose)

    os.remove(local_tarred_app)
    return remote_app_tar
  def test_appscale_in_two_node_virt_deployment(self):
    # pretend that the place we're going to put logs into doesn't exist
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False)

    # and mock out the mkdir operation
    flexmock(os)
    os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

    # next, mock out finding the login ip address
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "public1",
      "private_ip" : "private1",
      "jobs" : ["shadow", "login"]
    }]))
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # and slip in a fake appcontroller to report on the two IP addrs
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args(
      'the secret').and_return(json.dumps(['public1', 'public2']))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # fake the creation of the log directories locally
    os.should_receive('mkdir').with_args('/tmp/foobaz/public1').and_return()
    os.should_receive('mkdir').with_args('/tmp/foobaz/public2').and_return()

    # finally, fake the copying of the log files
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    argv = [
      "--keyname", self.keyname,
      "--location", "/tmp/foobaz"
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.gather_logs(options)
Пример #29
0
  def terminate_virtualized_cluster(cls, keyname, is_verbose):
    """Stops all API services running on all nodes in the currently running
    AppScale deployment.

    Args:
      keyname: The name of the SSH keypair used for this AppScale deployment.
      is_verbose: A bool that indicates if we should print the commands executed
        to stdout.
    """
    AppScaleLogger.log("Terminating appscale deployment with keyname {0}"
                       .format(keyname))
    time.sleep(2)

    shadow_host = LocalState.get_host_with_role(keyname, 'shadow')
    try:
      secret = LocalState.get_secret_key(keyname)
    except IOError:
      # We couldn't find the secret key: AppScale is most likely not
      # running.
      raise AppScaleException("Couldn't find AppScale secret key.")

    acc = AppControllerClient(shadow_host, secret)
    try:
      all_ips = acc.get_all_public_ips()
    except Exception as exception:
      AppScaleLogger.warn('Saw Exception while getting deployments IPs {0}'.
                          format(str(exception)))
      all_ips = LocalState.get_all_public_ips(keyname)

    threads = []
    for ip in all_ips:
      thread = threading.Thread(target=cls.stop_remote_appcontroller, args=(ip,
        keyname, is_verbose))
      thread.start()
      threads.append(thread)

    for thread in threads:
      thread.join()

    boxes_shut_down = 0
    is_running_regex = re.compile("appscale-controller stop")
    for ip in all_ips:
      AppScaleLogger.log("Shutting down AppScale API services at {0}".
                         format(ip))
      while True:
        remote_output = cls.ssh(ip, keyname, 'ps x', is_verbose)
        AppScaleLogger.verbose(remote_output, is_verbose)
        if not is_running_regex.match(remote_output):
          break
        time.sleep(0.3)
      boxes_shut_down += 1

    if boxes_shut_down != len(all_ips):
      raise AppScaleException("Couldn't terminate your AppScale deployment on"
                              " all machines - please do so manually.")

    AppScaleLogger.log("Terminated AppScale on {0} machines.".
                       format(boxes_shut_down))
Пример #30
0
    def test_describe_instances_with_two_nodes(self):
        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')  # set the fall-through

        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        fake_secret.should_receive('write').and_return()
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # mock out the SOAP call to the AppController and assume it succeeded
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
          .and_return(json.dumps(['public1', 'public2']))
        fake_appcontroller.should_receive('status').with_args('the secret') \
          .and_return('nothing interesting here') \
          .and_return('Database is at not-up-yet') \
          .and_return('Database is at 1.2.3.4')
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
          .and_return(fake_appcontroller)

        # mock out reading the locations.json file, and slip in our own json
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_nodes_json")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([
                {
                    "public_ip": "public1",
                    "private_ip": "private1",
                    "jobs": ["shadow", "login"]
                },
                {
                    "public_ip": "public2",
                    "private_ip": "private2",
                    "jobs": ["appengine"]
                },
            ]))
        fake_nodes_json.should_receive('write').and_return()
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)
        # assume that there are two machines running in our deployment

        argv = ["--keyname", self.keyname]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.describe_instances(options)
  def test_remove_app_and_app_is_running(self):
    # mock out reading from stdin, and assume the user says 'YES'
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_receive('raw_input').and_return('YES')

    # mock out reading the secret key
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('stop_app').with_args('blargapp',
      'the secret').and_return('OK')
    fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
      'the secret').and_return(True).and_return(True).and_return(False)
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "public1",
      "private_ip" : "private1",
      "jobs" : ["shadow", "login"]
    }]))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out calls to the UserAppServer and presume that the app does exist
    fake_userappserver = flexmock(name='fake_uaserver')
    fake_userappserver.should_receive('get_app_data').with_args(
      'blargapp', 'the secret').and_return(json.dumps({
        'hosts' : { '192.168.1.1' : { 'http' : '80', 'https' : '443' }}}))
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
      .and_return(fake_userappserver)

    argv = [
      "--appname", "blargapp",
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.remove_app(options)
Пример #32
0
  def create_user_accounts(cls, email, password, public_ip, keyname):
    """Registers two new user accounts with the UserAppServer.

    One account is the standard account that users log in with (via their
    e-mail address. The other is their XMPP account, so that they can log into
    any jabber-compatible service and send XMPP messages to their application
    (and receive them).

    Args:
      email: The e-mail address that should be registered for the user's
        standard account.
      password: The password that should be used for both the standard and XMPP
        accounts.
      public_ip: The location where the AppController can be found.
      keyname: The name of the SSH keypair used for this AppScale deployment.
    """
    acc = AppControllerClient(public_ip, LocalState.get_secret_key(keyname))

    is_new_user = False
    # first, create the standard account
    encrypted_pass = LocalState.encrypt_password(email, password)
    if acc.does_user_exist(email):
      AppScaleLogger.log("User {0} already exists, so not creating it again.".
        format(email))
    else:
      acc.create_user(email, encrypted_pass)
      is_new_user = True

    # next, create the XMPP account. if the user's e-mail is [email protected], then that
    # means their XMPP account name is a@login_ip
    username_regex = re.compile('\A(.*)@')
    username = username_regex.match(email).groups()[0]

    try:
      login_host = acc.get_property('login')['login']
    except KeyError:
      raise AppControllerException('login property not found')

    xmpp_user = "******".format(username, login_host)
    xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

    is_xmpp_user_exist = acc.does_user_exist(xmpp_user)

    if is_xmpp_user_exist and is_new_user:
      AppScaleLogger.log("XMPP User {0} conflict!".format(xmpp_user))

      generated_xmpp_username = LocalState.generate_xmpp_username(username)
      xmpp_user = "******".format(generated_xmpp_username, login_host)
      xmpp_pass = LocalState.encrypt_password(xmpp_user, password)

      acc.create_user(xmpp_user, xmpp_pass)
    elif is_xmpp_user_exist and not is_new_user:
      AppScaleLogger.log("XMPP User {0} already exists, so not creating it again.".format(xmpp_user))
    else:
      acc.create_user(xmpp_user, xmpp_pass)
    AppScaleLogger.log("Your XMPP username is {0}".format(xmpp_user))
Пример #33
0
  def test_update_local_metadata(self):
    # mock out getting all the ips in the deployment from the head node
    fake_soap = flexmock(name='fake_soap')
    fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['public1']))
    role_info = [{
        'public_ip' : 'public1',
        'private_ip' : 'private1',
        'jobs' : ['shadow', 'db_master']
    }]
    fake_soap.should_receive('get_role_info').with_args('the secret') \
      .and_return(json.dumps(role_info))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_soap)

    # mock out reading the secret key
    fake_secret = flexmock(name='fake_secret')
    fake_secret.should_receive('read').and_return('the secret')
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')
    builtins.should_receive('open').with_args(
      LocalState.get_secret_key_location('booscale'), 'r') \
      .and_return(fake_secret)

    # mock out writing the yaml file
    fake_locations_yaml = flexmock(name='fake_locations_yaml')
    fake_locations_yaml.should_receive('write').with_args(yaml.dump({
      'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG',
      'secret': 'the secret', 'infrastructure': 'ec2',
      'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra',
      'db_master': 'node-0', 'zone' : 'my-zone-1b'
    })).and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_yaml_location('booscale'), 'w') \
      .and_return(fake_locations_yaml)

    # and mock out writing the json file
    fake_locations_json = flexmock(name='fake_locations_json')
    fake_locations_json.should_receive('write').with_args(json.dumps(
      role_info)).and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location('booscale'), 'w') \
      .and_return(fake_locations_json)

    options = flexmock(name='options', table='cassandra', infrastructure='ec2',
      keyname='booscale', group='boogroup', zone='my-zone-1b')
    node_layout = NodeLayout(options={
      'min' : 1,
      'max' : 1,
      'infrastructure' : 'ec2',
      'table' : 'cassandra'
    })
    host = 'public1'
    instance_id = 'i-ABCDEFG'
    LocalState.update_local_metadata(options, node_layout, host, instance_id)
Пример #34
0
    def test_remove_app_and_app_is_running(self):
        # mock out reading from stdin, and assume the user says 'YES'
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_receive('raw_input').and_return('YES')

        # mock out reading the secret key
        builtins.should_call('open')  # set the fall-through

        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # mock out the SOAP call to the AppController and assume it succeeded
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('status').with_args('the secret') \
          .and_return('Database is at public1')
        fake_appcontroller.should_receive('stop_app').with_args(
            'blargapp', 'the secret').and_return('OK')
        fake_appcontroller.should_receive('is_app_running').with_args(
            'blargapp',
            'the secret').and_return(True).and_return(True).and_return(False)
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        # mock out reading the locations.json file, and slip in our own json
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_nodes_json")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([{
                "public_ip": "public1",
                "private_ip": "private1",
                "jobs": ["shadow", "login"]
            }]))
        fake_nodes_json.should_receive('write').and_return()
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out calls to the UserAppServer and presume that the app does exist
        fake_userappserver = flexmock(name='fake_uaserver')
        fake_userappserver.should_receive('get_app_data').with_args(
            'blargapp', 'the secret').and_return('\nnum_ports:2\n')
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
          .and_return(fake_userappserver)

        argv = ["--appname", "blargapp", "--keyname", self.keyname]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.remove_app(options)
Пример #35
0
  def test_update_local_metadata(self):
    # mock out getting all the ips in the deployment from the head node
    fake_soap = flexmock(name='fake_soap')
    fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['public1']))
    role_info = [{
        'public_ip' : 'public1',
        'private_ip' : 'private1',
        'jobs' : ['shadow', 'db_master']
    }]
    fake_soap.should_receive('get_role_info').with_args('the secret') \
      .and_return(json.dumps(role_info))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_soap)

    # mock out reading the secret key
    fake_secret = flexmock(name='fake_secret')
    fake_secret.should_receive('read').and_return('the secret')
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')
    builtins.should_receive('open').with_args(
      LocalState.get_secret_key_location('booscale'), 'r') \
      .and_return(fake_secret)

    # mock out writing the yaml file
    fake_locations_yaml = flexmock(name='fake_locations_yaml')
    fake_locations_yaml.should_receive('write').with_args(yaml.dump({
      'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG',
      'secret': 'the secret', 'infrastructure': 'ec2',
      'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra',
      'db_master': 'node-0', 'zone' : 'my-zone-1b'
    })).and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_yaml_location('booscale'), 'w') \
      .and_return(fake_locations_yaml)

    # and mock out writing the json file
    fake_locations_json = flexmock(name='fake_locations_json')
    fake_locations_json.should_receive('write').with_args(json.dumps(
      role_info)).and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location('booscale'), 'w') \
      .and_return(fake_locations_json)

    options = flexmock(name='options', table='cassandra', infrastructure='ec2',
      keyname='booscale', group='boogroup', zone='my-zone-1b')
    node_layout = NodeLayout(options={
      'min' : 1,
      'max' : 1,
      'infrastructure' : 'ec2',
      'table' : 'cassandra'
    })
    host = 'public1'
    instance_id = 'i-ABCDEFG'
    LocalState.update_local_metadata(options, node_layout, host, instance_id)
  def test_describe_instances_with_two_nodes(self):
    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['public1', 'public2']))
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('nothing interesting here') \
      .and_return('Database is at not-up-yet') \
      .and_return('Database is at 1.2.3.4')
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "public1",
      "private_ip" : "private1",
      "jobs" : ["shadow", "login"]
    },
    {
      "public_ip" : "public2",
      "private_ip" : "private2",
      "jobs" : ["appengine"]
    },
    ]))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)
    # assume that there are two machines running in our deployment

    argv = [
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.describe_instances(options)
Пример #37
0
    def test_reset_password_for_user_that_doesnt_exist(self):
        # put in a mock for reading the secret file
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')  # set the fall-through

        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # mock out reading the username and new password from the user
        builtins.should_receive('raw_input').and_return('*****@*****.**')
        flexmock(getpass)
        getpass.should_receive('getpass').and_return('the password')

        # mock out finding the login node's IP address from the json file
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_secret")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([{
                'public_ip': 'public1',
                'private_ip': 'private1',
                'jobs': ['login', 'db_master']
            }]))
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out grabbing the userappserver ip from an appcontroller
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('status').with_args('the secret') \
          .and_return('nothing interesting here') \
          .and_return('Database is at not-up-yet') \
          .and_return('Database is at public1')

        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str,
            'the secret').and_return('Error: user does not exist')
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
          .and_return(fake_userappserver)

        argv = ["--keyname", self.keyname]
        options = ParseArgs(argv, self.function).args
        self.assertRaises(SystemExit, AppScaleTools.reset_password, options)
Пример #38
0
  def remove_app(cls, options):
    """Instructs AppScale to no longer host the named application.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if not options.confirm:
      response = raw_input(
        'Are you sure you want to remove this application? (y/N) ')
      if response.lower() not in ['y', 'yes']:
        raise AppScaleException("Cancelled application removal.")

    login_host = LocalState.get_login_host(options.keyname)
    secret = LocalState.get_secret_key(options.keyname)
    acc = AppControllerClient(login_host, secret)

    if not acc.is_app_running(options.appname):
      raise AppScaleException("The given application is not currently running.")

    # Makes a call to the AppController to get all the stats and looks
    # through them for the http port the app can be reached on.
    http_port = None
    for _ in range(cls.MAX_RETRIES + 1):
      result = acc.get_all_stats()
      try:
        json_result = json.loads(result)
        apps_result = json_result['apps']
        current_app = apps_result[options.appname]
        http_port = current_app['http']
        if http_port:
          break
        time.sleep(cls.SLEEP_TIME)
      except (KeyError, ValueError):
        AppScaleLogger.verbose("Got json error from get_all_data result.",
            options.verbose)
        time.sleep(cls.SLEEP_TIME)
    if not http_port:
      raise AppScaleException(
        "Unable to get the serving port for the application.")

    acc.stop_app(options.appname)
    AppScaleLogger.log("Please wait for your app to shut down.")

    for _ in range(cls.MAX_RETRIES + 1):
      if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
        time.sleep(cls.SLEEP_TIME)
        AppScaleLogger.log("Waiting for {0} to terminate...".format(
          options.appname))
      else:
        AppScaleLogger.success("Done shutting down {0}.".format(
          options.appname))
        return
    AppScaleLogger.warn("App {0} may still be running.".format(
      options.appname))
  def test_reset_password_for_user_that_exists(self):
    # put in a mock for reading the secret file
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out reading the username and new password from the user
    builtins.should_receive('raw_input').and_return('*****@*****.**')
    flexmock(getpass)
    getpass.should_receive('getpass').and_return('the password')

    # mock out finding the login node's IP address from the json file
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_secret")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      'public_ip' : 'public1',
      'private_ip' : 'private1',
      'jobs' : ['login', 'db_master']
     }]))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out grabbing the userappserver ip from an appcontroller
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('nothing interesting here') \
      .and_return('Database is at not-up-yet') \
      .and_return('Database is at public1')

    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str, 'the secret').and_return('true')
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \
      .and_return(fake_userappserver)

    argv = [
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.reset_password(options)
Пример #40
0
    def get_params_from_args(self, args):
        """ Constructs a dict with only the parameters necessary to interact with
    Google Compute Engine (here, the client_secrets file and the image name).

    Args:
      args: A Namespace or dict that maps all of the arguments the user has
        invoked an AppScale command with their associated value.
    Returns:
      A dict containing the location of the client_secrets file and that name
      of the image to use in GCE.
    Raises:
      AgentConfigurationException: If the caller fails to specify a
        client_secrets file, or if it doesn't exist on the local filesystem.
    """
        if not isinstance(args, dict):
            args = vars(args)

        if not args.get('client_secrets') and not args.get('oauth2_storage'):
            raise AgentConfigurationException("Please specify a client_secrets " + \
              "file or a oauth2_storage file in your AppScalefile when running " + \
              "over Google Compute Engine.")

        credentials_file = args.get('client_secrets') or args.get(
            'oauth2_storage')
        full_credentials = os.path.expanduser(credentials_file)
        if not os.path.exists(full_credentials):
            raise AgentConfigurationException("Couldn't find your credentials " + \
              "at {0}".format(full_credentials))

        if args.get('client_secrets'):
            destination = LocalState.get_client_secrets_location(
                args['keyname'])
        elif args.get('oauth2_storage'):
            destination = LocalState.get_oauth2_storage_location(
                args['keyname'])

        shutil.copy(full_credentials, destination)

        params = {
            self.PARAM_GROUP: args['group'],
            self.PARAM_IMAGE_ID: args['machine'],
            self.PARAM_INSTANCE_TYPE: args['gce_instance_type'],
            self.PARAM_KEYNAME: args['keyname'],
            self.PARAM_PROJECT: args['project'],
            self.PARAM_ZONE: args['zone']
        }

        if args.get(self.PARAM_SECRETS):
            params[self.PARAM_SECRETS] = args.get(self.PARAM_SECRETS)
        elif args.get(self.PARAM_STORAGE):
            params[self.PARAM_STORAGE] = args.get(self.PARAM_STORAGE)

        params[self.PARAM_VERBOSE] = args.get('verbose', False)

        return params
Пример #41
0
    def configure_instance_security(self, parameters):
        """
    Setup Euca security keys and groups. Required input values are read from
    the parameters dictionary. More specifically, this method expects to
    find a 'keyname' parameter and a 'group' parameter in the parameters
    dictionary. Using these provided values, this method will create a new
    Euca key-pair and a security group. Security group will be granted permissions
    to access any port on the instantiated VMs. (Also see documentation for the
    BaseAgent class)

    Args:
      parameters  A dictionary of parameters
    """
        keyname = parameters[self.PARAM_KEYNAME]
        group = parameters[self.PARAM_GROUP]

        AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
          " is not already registered.")
        conn = self.open_connection(parameters)
        try:
            conn.get_key_pair(keyname)
            self.handle_failure(
                'SSH key found locally - please use a different keyname')
        except IndexError:  # in euca, this means the key doesn't exist
            pass

        security_groups = conn.get_all_security_groups()
        group_exists = False
        for security_group in security_groups:
            if security_group.name == group:
                self.handle_failure("Security group already exists - please use a " + \
                  "different group name")

        AppScaleLogger.log('Creating key pair: ' + keyname)
        key_pair = conn.create_key_pair(keyname)
        ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
        LocalState.write_key_file(ssh_key, key_pair.material)

        AppScaleLogger.log('Creating security group: {0}'.format(group))
        conn.create_security_group(group, 'AppScale security group')
        conn.authorize_security_group_deprecated(group,
                                                 from_port=1,
                                                 to_port=65535,
                                                 ip_protocol='udp',
                                                 cidr_ip='0.0.0.0/0')
        conn.authorize_security_group_deprecated(group,
                                                 from_port=1,
                                                 to_port=65535,
                                                 ip_protocol='tcp',
                                                 cidr_ip='0.0.0.0/0')
        conn.authorize_security_group_deprecated(group,
                                                 ip_protocol='icmp',
                                                 cidr_ip='0.0.0.0/0')

        return True
Пример #42
0
    def copy_deployment_credentials(cls, host, options):
        """Copies credentials needed to start the AppController and have it create
    other instances (in cloud deployments).

    Args:
      host: A str representing the machine (reachable from this computer) to
        copy our deployment credentials to.
      options: A Namespace that indicates which SSH keypair to use, and whether
        or not we are running in a cloud infrastructure.
    """
        local_secret_key = LocalState.get_secret_key_location(options.keyname)
        cls.scp(host, options.keyname, local_secret_key,
                '{}/secret.key'.format(cls.CONFIG_DIR), options.verbose)

        local_ssh_key = LocalState.get_key_path_from_name(options.keyname)
        cls.scp(host, options.keyname, local_ssh_key,
                '{}/ssh.key'.format(cls.CONFIG_DIR), options.verbose)

        LocalState.generate_ssl_cert(options.keyname, options.verbose)

        local_cert = LocalState.get_certificate_location(options.keyname)
        cls.scp(host, options.keyname, local_cert,
                '{}/certs/mycert.pem'.format(cls.CONFIG_DIR), options.verbose)

        local_private_key = LocalState.get_private_key_location(
            options.keyname)
        cls.scp(host, options.keyname, local_private_key,
                '{}/certs/mykey.pem'.format(cls.CONFIG_DIR), options.verbose)

        hash_id = subprocess.Popen([
            "openssl", "x509", "-hash", "-noout", "-in",
            LocalState.get_certificate_location(options.keyname)
        ],
                                   stdout=subprocess.PIPE).communicate()[0]
        symlink_cert = 'ln -fs {}/certs/mycert.pem /etc/ssl/certs/{}.0'.\
          format(cls.CONFIG_DIR, hash_id.rstrip())
        cls.ssh(host, options.keyname, symlink_cert, options.verbose)

        # In Google Compute Engine, we also need to copy over our client_secrets
        # file and the OAuth2 file that the user has approved for use with their
        # credentials, otherwise the AppScale VMs won't be able to interact with
        # GCE.
        if options.infrastructure and options.infrastructure == 'gce':
            secrets_location = LocalState.get_client_secrets_location(
                options.keyname)
            if not os.path.exists(secrets_location):
                raise AppScaleException(
                    '{} does not exist.'.format(secrets_location))
            secrets_type = GCEAgent.get_secrets_type(secrets_location)
            cls.scp(host, options.keyname, secrets_location,
                    '{}/client_secrets.json'.format(cls.CONFIG_DIR),
                    options.verbose)
            if secrets_type == CredentialTypes.OAUTH:
                local_oauth = LocalState.get_oauth2_storage_location(
                    options.keyname)
                cls.scp(host, options.keyname, local_oauth,
                        '{}/oauth2.dat'.format(cls.CONFIG_DIR),
                        options.verbose)
Пример #43
0
  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(options.location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {0}, already exists.".format(options.location))

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))

    try:
      all_ips = acc.get_all_public_ips()
    except socket.error:  # Occurs when the AppController has failed.
      AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
        "machines in this AppScale deployment. Using our locally cached " + \
        "info instead.")
      all_ips = LocalState.get_all_public_ips(options.keyname)

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(options.location)

    for ip in all_ips:
      # Get the logs from each node, and store them in our local directory
      local_dir = "{0}/{1}".format(options.location, ip)
      os.mkdir(local_dir)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
        local_dir, options.verbose)
      try:
        RemoteHelper.scp_remote_to_local(ip, options.keyname,
          '/var/log/cassandra', local_dir, options.verbose)
      except ShellException:
        pass

      try:
        RemoteHelper.scp_remote_to_local(ip, options.keyname,
          '/var/log/zookeeper', local_dir, options.verbose)
      except ShellException:
        pass

      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/kern.log',
        local_dir, options.verbose)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/syslog',
        local_dir, options.verbose)
    AppScaleLogger.success("Successfully copied logs to {0}".format(
      options.location))
Пример #44
0
  def configure_instance_security(self, parameters):
    """
    Setup Euca security keys and groups. Required input values are read from
    the parameters dictionary. More specifically, this method expects to
    find a 'keyname' parameter and a 'group' parameter in the parameters
    dictionary. Using these provided values, this method will create a new
    Euca key-pair and a security group. Security group will be granted permissions
    to access any port on the instantiated VMs. (Also see documentation for the
    BaseAgent class)

    Args:
      parameters  A dictionary of parameters
    """
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]

    AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
      " is not already registered.")
    conn = self.open_connection(parameters)
    try:
      conn.get_key_pair(keyname)
      self.handle_failure("SSH keyname {0} is already registered. Please " \
        "change the 'keyname' specified in your AppScalefile to a different " \
        "value, or erase it to have one automatically generated for you." \
        .format(keyname))
    except IndexError:  # in euca, this means the key doesn't exist
      pass

    security_groups = conn.get_all_security_groups()
    group_exists = False
    for security_group in security_groups:
      if security_group.name == group:
        self.handle_failure("Security group {0} is already registered. Please" \
          " change the 'group' specified in your AppScalefile to a different " \
          "value, or erase it to have one automatically generated for you." \
          .format(group))

    AppScaleLogger.log('Creating key pair: ' + keyname)
    key_pair = conn.create_key_pair(keyname)
    ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
    LocalState.write_key_file(ssh_key, key_pair.material)

    AppScaleLogger.log('Creating security group: {0}'.format(group))
    conn.create_security_group(group, 'AppScale security group')
    conn.authorize_security_group_deprecated(group, from_port=1,
      to_port=65535, ip_protocol='udp', cidr_ip='0.0.0.0/0')
    conn.authorize_security_group_deprecated(group, from_port=1,
      to_port=65535, ip_protocol='tcp', cidr_ip='0.0.0.0/0')
    conn.authorize_security_group_deprecated(group, from_port=-1,
      to_port=-1, ip_protocol='icmp', cidr_ip='0.0.0.0/0')

    return True
Пример #45
0
  def add_keypair(cls, options):
    """Sets up passwordless SSH login to the machines used in a virtualized
    cluster deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppScaleException: If any of the machines named in the ips_layout are
        not running, or do not have the SSH daemon running.
    """
    LocalState.require_ssh_commands(options.auto, options.verbose)
    LocalState.make_appscale_directory()

    path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
    if options.add_to_existing:
      public_key = path + ".pub"
      private_key = path
    else:
      public_key, private_key = LocalState.generate_rsa_key(options.keyname,
        options.verbose)

    if options.auto:
      if 'root_password' in options:
        AppScaleLogger.log("Using the provided root password to log into " + \
          "your VMs.")
        password = options.root_password
      else:
        AppScaleLogger.log("Please enter the password for the root user on" + \
          " your VMs:")
        password = getpass.getpass()

    node_layout = NodeLayout(options)
    if not node_layout.is_valid():
      raise BadConfigurationException("There were problems with your " + \
        "placement strategy: " + str(node_layout.errors()))

    all_ips = [node.public_ip for node in node_layout.nodes]
    for ip in all_ips:
      # first, make sure ssh is actually running on the host machine
      if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
        options.verbose):
        raise AppScaleException("SSH does not appear to be running at {0}. " \
          "Is the machine at {0} up and running? Make sure your IPs are " \
          "correct!".format(ip))

      # next, set up passwordless ssh
      AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
      if options.auto:
        LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
          private_key, password), options.verbose)
      else:
        LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
          options.verbose)

    AppScaleLogger.success("Generated a new SSH key for this deployment " + \
      "at {0}".format(private_key))
Пример #46
0
  def add_keypair(cls, options):
    """Sets up passwordless SSH login to the machines used in a virtualized
    cluster deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    Raises:
      AppScaleException: If any of the machines named in the ips_layout are
        not running, or do not have the SSH daemon running.
    """
    LocalState.require_ssh_commands(options.auto, options.verbose)
    LocalState.make_appscale_directory()

    path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
    if options.add_to_existing:
      public_key = path + ".pub"
      private_key = path
    else:
      public_key, private_key = LocalState.generate_rsa_key(options.keyname,
        options.verbose)

    if options.auto:
      if 'root_password' in options:
        AppScaleLogger.log("Using the provided root password to log into " + \
          "your VMs.")
        password = options.root_password
      else:
        AppScaleLogger.log("Please enter the password for the root user on" + \
          " your VMs:")
        password = getpass.getpass()

    node_layout = NodeLayout(options)
    if not node_layout.is_valid():
      raise BadConfigurationException("There were problems with your " + \
        "placement strategy: " + str(node_layout.errors()))

    all_ips = [node.public_ip for node in node_layout.nodes]
    for ip in all_ips:
      # first, make sure ssh is actually running on the host machine
      if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
        options.verbose):
        raise AppScaleException("SSH does not appear to be running at {0}. " \
          "Is the machine at {0} up and running? Make sure your IPs are " \
          "correct!".format(ip))

      # next, set up passwordless ssh
      AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
      if options.auto:
        LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
          private_key, password), options.verbose)
      else:
        LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
          options.verbose)

    AppScaleLogger.success("Generated a new SSH key for this deployment " + \
      "at {0}".format(private_key))
Пример #47
0
    def add_keypair(cls, options):
        """Sets up passwordless SSH login to the machines used in a virtualized
    cluster deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
        LocalState.require_ssh_commands(options.auto, options.verbose)
        LocalState.make_appscale_directory()

        path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
        if options.add_to_existing:
            public_key = path + ".pub"
            private_key = path
        else:
            public_key, private_key = LocalState.generate_rsa_key(
                options.keyname, options.verbose)

        if options.auto:
            if 'root_password' in options:
                AppScaleLogger.log("Using the provided root password to log into " + \
                  "your VMs.")
                password = options.root_password
            else:
                AppScaleLogger.log("Please enter the password for the root user on" + \
                  " your VMs:")
                password = getpass.getpass()

        node_layout = NodeLayout(options)
        if not node_layout.is_valid():
            raise BadConfigurationException("There were problems with your " + \
              "placement strategy: " + str(node_layout.errors()))

        all_ips = [node.public_ip for node in node_layout.nodes]
        for ip in all_ips:
            # first, set up passwordless ssh
            AppScaleLogger.log(
                "Executing ssh-copy-id for host: {0}".format(ip))
            if options.auto:
                LocalState.shell(
                    "{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
                                                  private_key, password),
                    options.verbose)
            else:
                LocalState.shell(
                    "ssh-copy-id -i {0} root@{1}".format(private_key, ip),
                    options.verbose)

            # next, copy over the ssh keypair we generate
            RemoteHelper.scp(ip, options.keyname, public_key,
                             '/root/.ssh/id_rsa.pub', options.verbose)
            RemoteHelper.scp(ip, options.keyname, private_key,
                             '/root/.ssh/id_rsa', options.verbose)

        AppScaleLogger.success("Generated a new SSH key for this deployment " + \
          "at {0}".format(private_key))
 def publish_api_list(cls, api_list, url, keyname):
   eager = EagerClient(LocalState.get_login_host(keyname),
     LocalState.get_secret_key(keyname))
   temp_api_list = []
   for api in api_list:
     temp_api_list.append(api.to_dict())
   result = eager.publish_api_list(temp_api_list, url)
   if result['success']:
     AppScaleLogger.log('{0} APIs published to API store.'.format(len(api_list)))
   else:
     AppScaleLogger.warn(result['reason'])
     if result.get('detail'):
       AppScaleLogger.warn(str(result['detail']))
 def perform_eager_validation(cls, app, keyname):
   eager = EagerClient(LocalState.get_login_host(keyname),
     LocalState.get_secret_key(keyname))
   AppScaleLogger.log('Running EAGER validations for application.')
   result = eager.validate_application_for_deployment(app.to_dict())
   if not result['success']:
     AppScaleLogger.log('Validation errors encountered: {0}'.format(result['reason']))
     if hasattr(result, 'detail'):
       errors = result['detail'].split('|')
       AppScaleLogger.log('Following error details are available:')
       for e in errors:
         AppScaleLogger.log('  * {0}'.format(e))
   return result['success']
Пример #50
0
  def test_copy_deployment_credentials_in_cloud(self):
    # mock out the scp'ing to public1 and assume they succeed
    subprocess.should_receive('Popen').with_args(re.compile('secret.key'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('ssh.key'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out generating the private key
    flexmock(M2Crypto.RSA)
    fake_rsa_key = flexmock(name='fake_rsa_key')
    fake_rsa_key.should_receive('save_key').with_args(
      LocalState.get_private_key_location('bookey'), None)
    M2Crypto.RSA.should_receive('gen_key').and_return(fake_rsa_key)

    flexmock(M2Crypto.EVP)
    fake_pkey = flexmock(name='fake_pkey')
    fake_pkey.should_receive('assign_rsa').with_args(fake_rsa_key).and_return()
    M2Crypto.EVP.should_receive('PKey').and_return(fake_pkey)

    # and mock out generating the certificate
    flexmock(M2Crypto.X509)
    fake_cert = flexmock(name='fake_x509')
    fake_cert.should_receive('set_pubkey').with_args(fake_pkey).and_return()
    fake_cert.should_receive('set_subject')
    fake_cert.should_receive('set_issuer_name')
    fake_cert.should_receive('set_not_before')
    fake_cert.should_receive('set_not_after')
    fake_cert.should_receive('sign').with_args(fake_pkey, md="sha256")
    fake_cert.should_receive('save_pem').with_args(
      LocalState.get_certificate_location('bookey'))
    M2Crypto.X509.should_receive('X509').and_return(fake_cert)

    # next, mock out copying the private key and certificate
    subprocess.should_receive('Popen').with_args(re.compile('mycert.pem'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('mykey.pem'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
      verbose=True)
    RemoteHelper.copy_deployment_credentials('public1', options)
    def test_make_appscale_directory_creation(self):
        # let's say that our ~/.appscale directory
        # does not exist
        os.path.should_receive('exists') \
          .with_args(LocalState.LOCAL_APPSCALE_PATH) \
          .and_return(False) \
          .once()

        # thus, mock out making the appscale dir
        os.should_receive('mkdir') \
          .with_args(LocalState.LOCAL_APPSCALE_PATH) \
          .and_return()

        LocalState.make_appscale_directory()
Пример #52
0
  def test_make_appscale_directory_creation(self):
    # let's say that our ~/.appscale directory
    # does not exist
    os.path.should_receive('exists') \
      .with_args(LocalState.LOCAL_APPSCALE_PATH) \
      .and_return(False) \
      .once()

    # thus, mock out making the appscale dir
    os.should_receive('mkdir') \
      .with_args(LocalState.LOCAL_APPSCALE_PATH) \
      .and_return()

    LocalState.make_appscale_directory()
Пример #53
0
  def copy_deployment_credentials(cls, host, options):
    """Copies credentials needed to start the AppController and have it create
    other instances (in cloud deployments).

    Args:
      host: A str representing the machine (reachable from this computer) to
        copy our deployment credentials to.
      options: A Namespace that indicates which SSH keypair to use, and whether
        or not we are running in a cloud infrastructure.
    """
    cls.scp(host, options.keyname, LocalState.get_secret_key_location(
      options.keyname), '/etc/appscale/secret.key', options.verbose)
    cls.scp(host, options.keyname, LocalState.get_key_path_from_name(
      options.keyname), '/etc/appscale/ssh.key', options.verbose)

    LocalState.generate_ssl_cert(options.keyname, options.verbose)
    cls.scp(host, options.keyname, LocalState.get_certificate_location(
      options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose)
    cls.scp(host, options.keyname, LocalState.get_private_key_location(
      options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose)

    AppScaleLogger.log("Copying over deployment credentials")
    cert = LocalState.get_certificate_location(options.keyname)
    private_key = LocalState.get_private_key_location(options.keyname)

    cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1',
      options.verbose)
    cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem",
      options.verbose)
    cls.scp(host, options.keyname, private_key,
      "/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
Пример #54
0
  def test_extract_app_to_dir(self):

    flexmock(os)
    os.should_receive('mkdir').and_return()
    flexmock(os.path)
    os.path.should_receive('abspath').with_args('relative/app.tar.gz')\
      .and_return('/tmp/relative/app.tar.gz')

    flexmock(LocalState)
    LocalState.should_receive('shell')\
      .with_args(re.compile('tar zxvf /tmp/relative/app.tar.gz'),False)\
      .and_return()

    LocalState.extract_app_to_dir('relative/app.tar.gz',False)
Пример #55
0
    def create_new_user(self,
                        email,
                        password,
                        response,
                        account_type='xmpp_user'):
        """ Creates a new user account, by making both a standard login and an
    XMPP login account.

    Args:
      email: A str containing the e-mail address of the new user.
      password: A str containing the cleartext password for the new user.
      response: A webapp2 response that the new user's logged in cookie
        should be set in.
    Returns:
      True, if the user account was successfully created.
    Raises:
      AppHelperException: If the user account could not be created.
    """
        try:
            uaserver = self.get_uaserver()
            # First, create the standard account.
            encrypted_pass = LocalState.encrypt_password(email, password)
            result = uaserver.commit_new_user(email, encrypted_pass,
                                              account_type, GLOBAL_SECRET_KEY)
            if result != 'true':
                raise AppHelperException(result)

            # Next, create the XMPP account. if the user's e-mail is [email protected], then that
            # means their XMPP account name is a@login_ip.
            username_regex = re.compile(self.USERNAME_FROM_EMAIL_REGEX)
            username = username_regex.match(email).groups()[0]
            xmpp_user = "******".format(username, self.get_login_host())
            xmpp_pass = LocalState.encrypt_password(xmpp_user, password)
            result = uaserver.commit_new_user(xmpp_user, xmpp_pass,
                                              account_type, GLOBAL_SECRET_KEY)
            if result != 'true':
                raise AppHelperException(result)

            # TODO: We may not even be using this token since the switch to
            # full proxy nginx. Investigate this.
            self.create_token(email, email)
            self.set_appserver_cookie(email, self.get_user_app_list(email),
                                      response)
        except AppHelperException as err:
            logging.exception(err)
            raise AppHelperException(str(err))
        except Exception as err:
            logging.exception(err)
            raise AppHelperException(str(err))
        return True
    def test_get_property(self):
        # put in a mock for reading the secret file
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')  # set the fall-through

        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # mock out finding the shadow node's IP address from the json file
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_secret")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([{
                'public_ip': 'public1',
                'private_ip': 'private1',
                'jobs': ['login', 'shadow']
            }]))
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out grabbing the userappserver ip from an appcontroller
        property_name = "name"
        property_value = "value"
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('set_property').with_args(
            property_name, property_value, 'the secret').and_return('OK')

        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        argv = [
            "--keyname", self.keyname, "--property_name", property_name,
            "--property_value", property_value
        ]
        options = ParseArgs(argv, self.function).args

        result = AppScaleTools.set_property(options)
        self.assertEqual(None, result)
Пример #57
0
    def login_user(self, email, password, response):
        """ Checks to see if the user has entered in a valid email and password,
    logging the user in if they have.

    Args:
      email: A str containing the e-mail address of the user to login.
      password: A str containing the cleartext password of the user to login.
      response: A webapp2 response that the new user's logged in cookie
        should be set in.
    Return:
      True if the user logged in successfully, and False otherwise.
    """
        user_data = self.query_user_data(email)
        server_re = re.search(self.USER_DATA_PASSWORD_REGEX, user_data)
        if not server_re:
            logging.error("Failed Login: {0} regex failed".format(email))
            return False
        server_pwd = server_re.group(1)
        encrypted_pass = LocalState.encrypt_password(email, password)
        if server_pwd != encrypted_pass:
            logging.info("Failed Login: {0} password mismatch".format(email))
            return False
        self.create_token(email, email)
        self.set_appserver_cookie(email, self.get_user_app_list(email),
                                  response)
        return True