def test_appscale_with_ips_layout_flag_but_no_copy_id(self):
    # assume that we have ssh-keygen but not ssh-copy-id
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('hash ssh-keygen'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('hash ssh-copy-id'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : public1
database: public1
zookeeper: public2
appengine:  public3
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    self.assertRaises(BadConfigurationException, AppScaleTools.add_keypair,
      options)
Пример #2
0
    def testTailWithIndexInBounds(self):
        # calling 'appscale tail 1 *' should tail from the second node
        # (nodes[1]). If there are two nodes in this deployment,
        # we should tail from it successfully
        appscale = AppScale()

        contents = {'keyname': 'boo'}
        yaml_dumped_contents = yaml.dump(contents)

        one = {'public_ip': 'blarg'}
        two = {'public_ip': 'blarg2'}
        nodes = {'node_info': [one, two]}
        nodes_contents = json.dumps(nodes)

        mock = self.addMockForAppScalefile(appscale, yaml_dumped_contents)
        (mock.should_receive('open').with_args(
            appscale.get_locations_json_file('boo')).and_return(
                flexmock(read=lambda: nodes_contents)))

        flexmock(subprocess)
        subprocess.should_receive('call').with_args([
            "ssh", "-o", "StrictHostkeyChecking=no", "-i",
            appscale.get_key_location('boo'), "root@blarg2",
            "tail -F /var/log/appscale/c*"
        ]).and_return().once()
        appscale.tail(1, "c*")
    def test_appscale_with_ips_layout_flag_but_no_copy_id(self):
        # assume that we have ssh-keygen but not ssh-copy-id
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('hash ssh-keygen'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('hash ssh-copy-id'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.failed)

        # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
        # machines to boot with those addresses (and that can mess up our tests).
        ips_layout = yaml.safe_load("""
master : public1
database: public1
zookeeper: public2
appengine:  public3
    """)

        argv = [
            "--ips_layout",
            base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname
        ]
        options = ParseArgs(argv, self.function).args
        self.assertRaises(BadConfigurationException, AppScaleTools.add_keypair,
                          options)
Пример #4
0
    def test_appscale_in_two_node_virt_deployment(self):
        # pretend that the place we're going to put logs into doesn't exist
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
            False)

        # and mock out the mkdir operation
        flexmock(os)
        os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

        # next, mock out finding the login ip address
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_nodes_json")
        fake_nodes_json.should_receive('read').and_return(
            json.dumps([{
                "public_ip": "public1",
                "private_ip": "private1",
                "jobs": ["shadow", "login"]
            }]))
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # and slip in a fake appcontroller to report on the two IP addrs
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('get_all_public_ips').with_args(
            'the secret').and_return(json.dumps(['public1', 'public2']))
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        # fake the creation of the log directories locally
        os.should_receive('mkdir').with_args(
            '/tmp/foobaz/public1').and_return()
        os.should_receive('mkdir').with_args(
            '/tmp/foobaz/public2').and_return()

        # finally, fake the copying of the log files
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.gather_logs(options)
Пример #5
0
  def testTailWithIndexInBounds(self):
    # calling 'appscale tail 1 *' should tail from the second node
    # (nodes[1]). If there are two nodes in this deployment,
    # we should tail from it successfully
    appscale = AppScale()

    contents = { 'keyname' : 'boo' }
    yaml_dumped_contents = yaml.dump(contents)

    one = {
      'public_ip' : 'blarg'
    }
    two = {
      'public_ip' : 'blarg2'
    }
    nodes = {'node_info': [one, two]}
    nodes_contents = json.dumps(nodes)

    mock = self.addMockForAppScalefile(appscale, yaml_dumped_contents)
    (mock.should_receive('open')
      .with_args(appscale.get_locations_json_file('boo'))
      .and_return(flexmock(read=lambda: nodes_contents)))

    flexmock(subprocess)
    subprocess.should_receive('call').with_args(["ssh", "-o",
      "StrictHostkeyChecking=no", "-i", appscale.get_key_location('boo'),
      "root@blarg2", "tail -F /var/log/appscale/c*"]).and_return().once()
    appscale.tail(1, "c*")
  def test_start_head_node_in_cloud_but_ami_not_appscale(self):
    # mock out our attempts to find /etc/appscale and presume it doesn't exist
    subprocess.should_receive('Popen').with_args(re.compile('/etc/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed)

    self.assertRaises(AppScaleException, RemoteHelper.start_head_node,
      self.options, self.node_layout)
  def test_appscale_in_two_node_virt_deployment(self):
    # pretend that the place we're going to put logs into doesn't exist
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False)

    # and mock out the mkdir operation
    flexmock(os)
    os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

    # next, mock out finding the login ip address
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "public1",
      "private_ip" : "private1",
      "jobs" : ["shadow", "login"]
    }]))
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # and slip in a fake appcontroller to report on the two IP addrs
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args(
      'the secret').and_return(json.dumps(['public1', 'public2']))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # fake the creation of the log directories locally
    os.should_receive('mkdir').with_args('/tmp/foobaz/public1').and_return()
    os.should_receive('mkdir').with_args('/tmp/foobaz/public2').and_return()

    # finally, fake the copying of the log files
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    argv = [
      "--keyname", self.keyname,
      "--location", "/tmp/foobaz"
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.gather_logs(options)
  def test_copy_deployment_credentials_in_cloud(self):
    # mock out the scp'ing to public1 and assume they succeed
    subprocess.should_receive('Popen').with_args(re.compile('secret.key'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('ssh.key'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out generating the private key
    flexmock(M2Crypto.RSA)
    fake_rsa_key = flexmock(name='fake_rsa_key')
    fake_rsa_key.should_receive('save_key').with_args(
      LocalState.get_private_key_location('bookey'), None)
    M2Crypto.RSA.should_receive('gen_key').and_return(fake_rsa_key)

    flexmock(M2Crypto.EVP)
    fake_pkey = flexmock(name='fake_pkey')
    fake_pkey.should_receive('assign_rsa').with_args(fake_rsa_key).and_return()
    M2Crypto.EVP.should_receive('PKey').and_return(fake_pkey)

    # and mock out generating the certificate
    flexmock(M2Crypto.X509)
    fake_cert = flexmock(name='fake_x509')
    fake_cert.should_receive('set_pubkey').with_args(fake_pkey).and_return()
    fake_cert.should_receive('set_subject')
    fake_cert.should_receive('set_issuer_name')
    fake_cert.should_receive('set_not_before')
    fake_cert.should_receive('set_not_after')
    fake_cert.should_receive('sign').with_args(fake_pkey, md="sha256")
    fake_cert.should_receive('save_pem').with_args(
      LocalState.get_certificate_location('bookey'))
    M2Crypto.X509.should_receive('X509').and_return(fake_cert)

    # next, mock out copying the private key and certificate
    subprocess.should_receive('Popen').with_args(re.compile('mycert.pem'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('mykey.pem'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
      verbose=True)
    RemoteHelper.copy_deployment_credentials('public1', options)
  def test_rsync_files_from_dir_that_does_exist(self):
    # if the user specifies that we should copy from a directory that does
    # exist, and has all the right directories in it, we should succeed
    flexmock(os.path)
    os.path.should_receive('exists').with_args(re.compile(
      '/tmp/booscale-local/')).and_return(True)

    # assume the rsyncs succeed
    subprocess.should_receive('Popen').with_args(re.compile('rsync'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    RemoteHelper.rsync_files('public1', 'booscale', '/tmp/booscale-local',
      False)
Пример #10
0
  def test_start_remote_appcontroller(self):
    # mock out removing the old json file
    subprocess.should_receive('Popen').with_args(re.compile('rm -rf'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # assume we started god on public1 fine
    subprocess.should_receive('Popen').with_args(re.compile('god &'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # also assume that we scp'ed over the god config file fine
    subprocess.should_receive('Popen').with_args(re.compile('appcontroller'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and assume we started the AppController on public1 fine
    subprocess.should_receive('Popen').with_args(re.compile('god load'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # finally, assume the appcontroller comes up after a few tries
    # assume that ssh comes up on the third attempt
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      AppControllerClient.PORT)).and_raise(Exception) \
      .and_raise(Exception).and_return(None)
    socket.should_receive('socket').and_return(fake_socket)

    RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)
Пример #11
0
  def test_copy_local_metadata(self):
    # mock out the copying of the two files
    subprocess.should_receive('Popen').with_args(re.compile(
      'locations-bookey.[yaml|json]'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and mock out copying the secret file
    subprocess.should_receive('Popen').with_args(re.compile(
      'bookey.secret'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    RemoteHelper.copy_local_metadata('public1', 'bookey', False)
Пример #12
0
  def test_start_head_node_in_cloud_but_using_unsupported_database(self):
    # mock out our attempts to find /etc/appscale and presume it does exist
    subprocess.should_receive('Popen').with_args(re.compile('/etc/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out our attempts to find /etc/appscale/version and presume it does
    # exist
    subprocess.should_receive('Popen').with_args(re.compile(
      '/etc/appscale/{0}'.format(APPSCALE_VERSION)),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # finally, put in a mock indicating that the database the user wants
    # isn't supported
    subprocess.should_receive('Popen').with_args(re.compile(
      '/etc/appscale/{0}/{1}'.format(APPSCALE_VERSION, 'cassandra')),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed)

    self.assertRaises(AppScaleException, RemoteHelper.start_head_node,
      self.options, self.node_layout)
  def test_upload_app_successfully(self):
    # add in mocks so that there is an app.yaml, but with no appid set
    flexmock(os.path)
    os.path.should_call('exists')
    app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
    os.path.should_receive('exists').with_args(app_yaml_location) \
      .and_return(True)

    # mock out reading the app.yaml file
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    fake_app_yaml = flexmock(name="fake_app_yaml")
    fake_app_yaml.should_receive('read').and_return(yaml.dump({
      'application' : 'baz',
      'runtime' : 'python27'
    }))
    builtins.should_receive('open').with_args(app_yaml_location, 'r') \
      .and_return(fake_app_yaml)

    # Mock out service host and port
    app_data = {'owner' : '*****@*****.**',
      'hosts' : {'192.168.1.1' : { 'http' : 8080, 'https' : 4380 }}}
    app_stats_data = {'apps': {'baz': {'http': 8080, 'language': 'python27',
      'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('done_uploading').with_args('baz',
      '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return()
    fake_appcontroller.should_receive('update').with_args(['baz'],
      'the secret').and_return()
    fake_appcontroller.should_receive('is_app_running').with_args('baz',
      'the secret').and_return(False).and_return(True)
    fake_appcontroller.should_receive('does_user_exist').with_args(
      '*****@*****.**', 'the secret').and_return('true')
    fake_appcontroller.should_receive('does_user_exist').with_args(
      'a@public1', 'the secret').and_return('true')
    fake_appcontroller.should_receive('does_app_exist').with_args(
      'baz', 'the secret').and_return(json.dumps(app_data))
    fake_appcontroller.should_receive('get_app_data').with_args(
      'baz', 'the secret').and_return(json.dumps(app_data))
    fake_appcontroller.should_receive('get_all_stats').with_args(
      'the secret').and_return(json.dumps(app_stats_data))
    fake_appcontroller.should_receive('reserve_app_id').with_args(
      '*****@*****.**','baz','python27','the secret').and_return("true")
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
    flexmock(getpass)
    getpass.should_receive('getpass').and_return('aaaaaa')

    # mock out making the remote app directory
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and mock out tarring and copying the app
    subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile(
      '/tmp/appscale-app-baz.tar.gz'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # as well as removing the tar'ed app once we're done copying it
    flexmock(os)
    os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
      .and_return()

    # and slap in a mock that says the app comes up after waiting for it
    # three times
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      8080)).and_raise(Exception).and_raise(Exception) \
      .and_return(None)
    flexmock(socket)
    socket.should_receive('socket').and_return(fake_socket)

    argv = [
      "--keyname", self.keyname,
      "--file", self.app_dir
    ]
    options = ParseArgs(argv, self.function).args
    (host, port) = AppScaleTools.upload_app(options)
    self.assertEquals('public1', host)
    self.assertEquals(8080, port) 
  def test_terminate_in_virtual_cluster_and_succeeds(self):
    # let's say that there is a locations.yaml file, which means appscale is
    # running, so we should terminate the services on each box
    flexmock(os.path)
    os.path.should_call('exists')  # set up the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_yaml_location(self.keyname)).and_return(True)

    # mock out reading the locations.yaml file, and pretend that we're on
    # a virtualized cluster
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')

    fake_yaml_file = flexmock(name='fake_file')
    fake_yaml_file.should_receive('read').and_return(yaml.dump({
      'infrastructure' : 'xen'
    }))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_yaml_location(self.keyname), 'r') \
      .and_return(fake_yaml_file)

    # mock out reading the json file, and pretend that we're running in a
    # two node deployment
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_json_file = flexmock(name='fake_file')
    fake_json_file.should_receive('read').and_return(json.dumps([
      {
        'public_ip' : 'public1',
        'jobs' : ['shadow']
      },
      {
        'public_ip' : 'public2',
        'jobs' : ['appengine']
      }
    ]))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_json_file)

    # and slip in a fake secret file
    fake_secret_file = flexmock(name='fake_file')
    fake_secret_file.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(
      LocalState.get_secret_key_location(self.keyname), 'r') \
      .and_return(fake_secret_file)

    # mock out talking to the appcontroller, and assume that it tells us there
    # there are still two machines in this deployment
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['public1', 'public2']))

    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # and mock out the ssh call to kill the remote appcontroller, assuming that
    # it fails the first time and passes the second
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('controller stop'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed).and_return(self.success)

    # next, mock out our checks to see how the stopping process is going and
    # assume that it has stopped
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('ps x'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # finally, mock out removing the yaml file, json file, and secret key from
    # this machine
    flexmock(os)
    os.should_receive('remove').with_args(
      LocalState.get_locations_yaml_location(self.keyname)).and_return()
    os.should_receive('remove').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return()
    os.should_receive('remove').with_args(
      LocalState.get_secret_key_location(self.keyname)).and_return()

    argv = [
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.terminate_instances(options)
    def test_appscale_in_two_node_virt_deployment(self):
        # pretend that the place we're going to put logs into doesn't exist
        flexmock(os.path)
        os.path.should_call("exists")  # set the fall-through
        os.path.should_receive("exists").with_args("/tmp/foobaz").and_return(False)

        # and mock out the mkdir operation
        flexmock(os)
        os.should_receive("mkdir").with_args("/tmp/foobaz").and_return()

        # next, mock out finding the login ip address
        os.path.should_receive("exists").with_args(LocalState.get_locations_json_location(self.keyname)).and_return(
            True
        )

        fake_nodes_json = flexmock(name="fake_nodes_json")
        fake_nodes_json.should_receive("read").and_return(
            json.dumps({"node_info": [{"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"]}]})
        )
        builtins = flexmock(sys.modules["__builtin__"])
        builtins.should_call("open")
        builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return(
            fake_nodes_json
        )

        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive("read").and_return("the secret")
        builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)

        # and slip in a fake appcontroller to report on the two IP addrs
        fake_appcontroller = flexmock(name="fake_appcontroller")
        fake_appcontroller.should_receive("get_all_public_ips").with_args("the secret").and_return(
            json.dumps(["public1", "public2"])
        )
        flexmock(SOAPpy)
        SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_appcontroller)

        # fake the creation of the log directories locally
        os.should_receive("mkdir").with_args("/tmp/foobaz/public1").and_return()
        os.should_receive("mkdir").with_args("/tmp/foobaz/public1/cassandra")
        os.should_receive("mkdir").with_args("/tmp/foobaz/public2").and_return()
        os.should_receive("mkdir").with_args("/tmp/foobaz/public2/cassandra")

        # finally, fake the copying of the log files
        flexmock(subprocess)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/appscale"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/kern.log*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/monit*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/nginx"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/syslog*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/var/log/zookeeper"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)
        subprocess.should_receive("Popen").with_args(
            re.compile("/opt/cassandra/cassandra/logs"),
            shell=True,
            stdout=self.fake_temp_file,
            stderr=subprocess.STDOUT,
        ).and_return(self.success)

        argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.gather_logs(options)
    def test_appscale_with_ips_layout_flag_and_success(self):
        # assume that ssh is running on each machine
        fake_socket = flexmock(name='socket')
        fake_socket.should_receive('connect').with_args(('1.2.3.4', 22)) \
          .and_return(None)
        fake_socket.should_receive('connect').with_args(('1.2.3.5', 22)) \
          .and_return(None)
        fake_socket.should_receive('connect').with_args(('1.2.3.6', 22)) \
          .and_return(None)

        flexmock(socket)
        socket.should_receive('socket').and_return(fake_socket)

        # assume that we have ssh-keygen and ssh-copy-id
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('which ssh-keygen'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('which ssh-copy-id'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # assume that we have a ~/.appscale
        flexmock(os.path)
        os.path.should_call('exists')
        os.path.should_receive('exists').with_args(LocalState.LOCAL_APPSCALE_PATH) \
          .and_return(True)

        # and assume that we don't have public and private keys already made
        path = LocalState.LOCAL_APPSCALE_PATH + self.keyname
        public_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname + '.pub'
        private_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname + '.key'

        os.path.should_receive('exists').with_args(public_key).and_return(
            False)
        os.path.should_receive('exists').with_args(private_key).and_return(
            False)

        # next, assume that ssh-keygen ran fine
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('ssh-keygen'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # assume that we can rename the private key
        flexmock(shutil)
        shutil.should_receive('copy').with_args(path, private_key).and_return()

        # finally, assume that we can chmod 0600 those files fine
        flexmock(os)
        os.should_receive('chmod').with_args(public_key, 0600).and_return()
        os.should_receive('chmod').with_args(path, 0600).and_return()

        # and assume that we can ssh-copy-id to each of the three IPs below
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('ssh-copy-id'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # also, we should be able to copy over our new public and private keys fine
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('id_rsa[.pub]?'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
        # machines to boot with those addresses (and that can mess up our tests).
        ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.5
appengine: 1.2.3.6
    """)

        argv = [
            "--ips_layout",
            base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname
        ]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.add_keypair(options)
    def test_upload_app_when_app_exists_on_virt_cluster(self):
        # we do let you upload an app if it's already running

        # add in mocks so that there is an app.yaml with an appid set
        flexmock(os.path)
        os.path.should_call('exists')
        app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
        os.path.should_receive('exists').with_args(app_yaml_location) \
          .and_return(True)

        # mock out reading the app.yaml file
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')  # set the fall-through

        fake_app_yaml = flexmock(name="fake_app_yaml")
        fake_app_yaml.should_receive('read').and_return(
            yaml.dump({
                'application': 'baz',
                'runtime': 'python27'
            }))
        builtins.should_receive('open').with_args(app_yaml_location, 'r') \
          .and_return(fake_app_yaml)

        # mock out the SOAP call to the AppController and assume it succeeded
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('status').with_args('the secret') \
          .and_return('Database is at public1')
        fake_appcontroller.should_receive('done_uploading').with_args(
            'baz', '/opt/appscale/apps/baz.tar.gz',
            'the secret').and_return('OK')
        fake_appcontroller.should_receive('update').with_args(
            ['baz'], 'the secret').and_return('OK')
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
        fake_userappserver.should_receive('does_user_exist').with_args(
            'a@public1', 'the secret').and_return('false')
        fake_userappserver.should_receive('commit_new_user').with_args(
            '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
        fake_userappserver.should_receive('commit_new_user').with_args(
            'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
        fake_userappserver.should_receive('get_app_data').with_args(
            'baz', 'the secret').and_return(app_data)
        SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
        flexmock(getpass)
        getpass.should_receive('getpass').and_return('aaaaaa')

        # mock out making the remote app directory
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # and mock out tarring and copying the app
        subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        subprocess.should_receive('Popen').with_args(re.compile(
          '/tmp/appscale-app-baz.tar.gz'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # as well as removing the tar'ed app once we're done copying it
        flexmock(os)
        os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
          .and_return()

        # and slap in a mock that says the app comes up after waiting for it
        # three times
        fake_socket = flexmock(name='fake_socket')
        fake_socket.should_receive('connect').with_args(('public1',
          8080)).and_raise(Exception).and_raise(Exception) \
          .and_return(None)
        flexmock(socket)
        socket.should_receive('socket').and_return(fake_socket)

        argv = ["--keyname", self.keyname, "--file", self.app_dir]
        options = ParseArgs(argv, self.function).args
        (host, port) = AppScaleTools.upload_app(options)
        self.assertEquals('public1', host)
        self.assertEquals(8080, port)
Пример #18
0
    def test_upload_app_successfully(self):
        # add in mocks so that there is an app.yaml, but with no appid set
        flexmock(os.path)
        os.path.should_call('exists')
        app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
        os.path.should_receive('exists').with_args(app_yaml_location) \
          .and_return(True)

        # mock out reading the app.yaml file
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')  # set the fall-through

        fake_app_yaml = flexmock(name="fake_app_yaml")
        fake_app_yaml.should_receive('read').and_return(
            yaml.dump({
                'application': 'baz',
                'runtime': 'python27'
            }))
        builtins.should_receive('open').with_args(app_yaml_location, 'r') \
          .and_return(fake_app_yaml)

        # Mock out service host and port
        app_data = {
            'owner': '*****@*****.**',
            'hosts': {
                '192.168.1.1': {
                    'http': 8080,
                    'https': 4380
                }
            }
        }
        app_stats_data = {
            'apps': {
                'baz': {
                    'http': 8080,
                    'language': 'python27',
                    'total_reqs': 'no_change',
                    'appservers': 1,
                    'https': 4380,
                    'reqs_enqueued': None
                }
            }
        }

        # mock out the SOAP call to the AppController and assume it succeeded
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('status').with_args('the secret') \
          .and_return('Database is at public1')
        fake_appcontroller.should_receive('done_uploading').with_args(
            'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return()
        fake_appcontroller.should_receive('update').with_args(
            ['baz'], 'the secret').and_return()
        fake_appcontroller.should_receive('does_user_exist').with_args(
            '*****@*****.**', 'the secret').and_return('true')
        fake_appcontroller.should_receive('does_user_exist').with_args(
            'a@public1', 'the secret').and_return('true')
        fake_appcontroller.should_receive('does_app_exist').with_args(
            'baz', 'the secret').and_return(json.dumps(app_data))
        fake_appcontroller.should_receive('get_app_data').with_args(
            'baz', 'the secret').and_return(json.dumps(app_data))
        fake_appcontroller.should_receive('get_all_stats').with_args(
            'the secret').and_return(json.dumps(app_stats_data))
        fake_appcontroller.should_receive('reserve_app_id').with_args(
            '*****@*****.**', 'baz', 'python27', 'the secret').and_return("true")
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
        flexmock(getpass)
        getpass.should_receive('getpass').and_return('aaaaaa')

        # mock out making the remote app directory
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # and mock out tarring and copying the app
        subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        subprocess.should_receive('Popen').with_args(re.compile(
          '/tmp/appscale-app-baz.tar.gz'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # as well as removing the tar'ed app once we're done copying it
        flexmock(os)
        os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
          .and_return()

        # and slap in a mock that says the app comes up after waiting for it
        # three times
        fake_socket = flexmock(name='fake_socket')
        fake_socket.should_receive('connect').with_args(('public1',
          8080)).and_raise(Exception).and_raise(Exception) \
          .and_return(None)
        flexmock(socket)
        socket.should_receive('socket').and_return(fake_socket)

        argv = ["--keyname", self.keyname, "--file", self.app_dir]
        options = ParseArgs(argv, self.function).args
        (host, port) = AppScaleTools.upload_app(options)
        self.assertEquals('public1', host)
        self.assertEquals(8080, port)
    def test_terminate_in_virtual_cluster_and_succeeds(self):
        # let's say that there is a locations.yaml file, which means appscale is
        # running, so we should terminate the services on each box
        flexmock(os.path)
        os.path.should_call("exists")  # set up the fall-through
        os.path.should_receive("exists").with_args(LocalState.get_secret_key_location(self.keyname)).and_return(True)

        # mock out reading the locations.yaml file, and pretend that we're on
        # a virtualized cluster
        builtins = flexmock(sys.modules["__builtin__"])
        builtins.should_call("open")

        fake_yaml_file = flexmock(name="fake_file")
        fake_yaml_file.should_receive("read").and_return(yaml.dump({"infrastructure": "xen"}))
        builtins.should_receive("open").with_args(LocalState.get_locations_yaml_location(self.keyname), "r").and_return(
            fake_yaml_file
        )

        # mock out reading the json file, and pretend that we're running in a
        # two node deployment
        os.path.should_receive("exists").with_args(LocalState.get_locations_json_location(self.keyname)).and_return(
            True
        )

        fake_json_file = flexmock(name="fake_file")
        fake_json_file.should_receive("read").and_return(
            json.dumps([{"public_ip": "public1", "jobs": ["shadow"]}, {"public_ip": "public2", "jobs": ["appengine"]}])
        )
        builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return(
            fake_json_file
        )

        # and slip in a fake secret file
        fake_secret_file = flexmock(name="fake_file")
        fake_secret_file.should_receive("read").and_return("the secret")
        builtins.should_receive("open").with_args(LocalState.get_secret_key_location(self.keyname), "r").and_return(
            fake_secret_file
        )

        # mock out talking to the appcontroller, and assume that it tells us there
        # there are still two machines in this deployment
        fake_appcontroller = flexmock(name="fake_appcontroller")
        fake_appcontroller.should_receive("get_all_public_ips").with_args("the secret").and_return(
            json.dumps(["public1", "public2"])
        )

        flexmock(SOAPpy)
        SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_appcontroller)

        # and mock out the ssh call to kill the remote appcontroller, assuming that
        # it fails the first time and passes the second
        flexmock(subprocess)
        subprocess.should_receive("Popen").with_args(
            re.compile("controller stop"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.failed).and_return(self.success)

        # next, mock out our checks to see how the stopping process is going and
        # assume that it has stopped
        flexmock(subprocess)
        subprocess.should_receive("Popen").with_args(
            re.compile("ps x"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
        ).and_return(self.success)

        # finally, mock out removing the yaml file, json file, and secret key from
        # this machine
        flexmock(os)
        os.should_receive("remove").with_args(LocalState.get_locations_yaml_location(self.keyname)).and_return()
        os.should_receive("remove").with_args(LocalState.get_locations_json_location(self.keyname)).and_return()
        os.should_receive("remove").with_args(LocalState.get_secret_key_location(self.keyname)).and_return()

        # also mock out asking the user for confirmation on shutting down
        # their cloud
        builtins.should_receive("raw_input").and_return("yes")

        argv = ["--keyname", self.keyname]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.terminate_instances(options)
  def test_appscale_in_two_node_virt_deployment(self):
    # pretend that the place we're going to put logs into doesn't exist
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False)

    # and mock out the mkdir operation
    flexmock(os)
    os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

    # next, mock out finding the login ip address
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    nodes_info = {
      "node_info": [
        {
          "public_ip": "public1",
          "private_ip": "private1",
          "roles": ["load_balancer", "taskqueue_master", "zookeeper",
                    "db_master", "taskqueue", "shadow"]
        }, {
          "public_ip": "public2",
          "private_ip": "private2",
          "roles": ["memcache", "appengine", "zookeeper"]
        }, {
          "public_ip": "public3",
          "private_ip": "private3",
          "roles": ["memcache", "appengine"]
        },
      ]
    }
    fake_nodes_json.should_receive('read').and_return(json.dumps(nodes_info))
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # and slip in a fake appcontroller to report on the two IP addrs
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args(
      'the secret').and_return(json.dumps(['public1', 'public2', 'public3']))
    fake_appcontroller.should_receive('get_role_info').with_args(
      'the secret').and_return(json.dumps(nodes_info['node_info']))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # fake the creation of the log directories locally
    flexmock(utils)
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/private-ips')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public1')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/cassandra')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public2')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/cassandra')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public3')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/cassandra')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/load_balancer')
    utils.should_receive('mkdir').with_args(
      '/tmp/foobaz/symlinks/taskqueue_master')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/zookeeper')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/db_master')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/taskqueue')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/memcache')
    utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/appengine')

    # fake creation of symlink to for friendly navigation
    links_mapping = {
      '../../public1': [
        '/tmp/foobaz/symlinks/private-ips/private1',
        '/tmp/foobaz/symlinks/load_balancer/public1',
        '/tmp/foobaz/symlinks/taskqueue_master/public1',
        '/tmp/foobaz/symlinks/zookeeper/public1',
        '/tmp/foobaz/symlinks/db_master/public1',
        '/tmp/foobaz/symlinks/taskqueue/public1',
        '/tmp/foobaz/symlinks/shadow/public1',
      ],
      '../../public2': [
        '/tmp/foobaz/symlinks/private-ips/private2',
        '/tmp/foobaz/symlinks/zookeeper/public2',
        '/tmp/foobaz/symlinks/appengine/public2',
        '/tmp/foobaz/symlinks/memcache/public2',
      ],
      '../../public3': [
        '/tmp/foobaz/symlinks/private-ips/private3',
        '/tmp/foobaz/symlinks/appengine/public3',
        '/tmp/foobaz/symlinks/memcache/public3',
      ]
    }
    for original_dir, expected_links in links_mapping.iteritems():
      for expected_link in expected_links:
        os.should_receive('symlink').with_args(original_dir, expected_link)

    # finally, fake the copying of the log files
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(
      re.compile('/var/log/haproxy*'), shell=True, stdout=self.fake_temp_file,
      stderr=subprocess.STDOUT).and_return(self.success)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(
      re.compile('/var/log/rabbitmq'), shell=True, stdout=self.fake_temp_file,
      stderr=subprocess.STDOUT).and_return(self.success)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)
    subprocess.should_receive('Popen').with_args(
      re.compile('/opt/cassandra/cassandra/logs'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
    ).and_return(self.success)

    argv = [
      "--keyname", self.keyname,
      "--location", "/tmp/foobaz"
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.gather_logs(options)
Пример #21
0
    def test_appscale_in_two_node_virt_deployment(self):
        # pretend that the place we're going to put logs into doesn't exist
        flexmock(os.path)
        os.path.should_call('exists')  # set the fall-through
        os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
            False)

        # and mock out the mkdir operation
        flexmock(os)
        os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()

        # next, mock out finding the login ip address
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_nodes_json = flexmock(name="fake_nodes_json")
        nodes_info = {
            "node_info": [
                {
                    "public_ip":
                    "public1",
                    "private_ip":
                    "private1",
                    "jobs": [
                        "load_balancer", "taskqueue_master", "zookeeper",
                        "db_master", "taskqueue", "shadow", "login"
                    ]
                },
                {
                    "public_ip": "public2",
                    "private_ip": "private2",
                    "jobs": ["memcache", "appengine", "zookeeper"]
                },
                {
                    "public_ip": "public3",
                    "private_ip": "private3",
                    "jobs": ["memcache", "appengine"]
                },
            ]
        }
        fake_nodes_json.should_receive('read').and_return(
            json.dumps(nodes_info))
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_nodes_json)

        # mock out writing the secret key to ~/.appscale, as well as reading it
        # later
        secret_key_location = LocalState.get_secret_key_location(self.keyname)
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(secret_key_location, 'r') \
          .and_return(fake_secret)

        # and slip in a fake appcontroller to report on the two IP addrs
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('get_all_public_ips').with_args(
            'the secret').and_return(
                json.dumps(['public1', 'public2', 'public3']))
        fake_appcontroller.should_receive('get_role_info').with_args(
            'the secret').and_return(json.dumps(nodes_info['node_info']))
        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        # fake the creation of the log directories locally
        flexmock(utils)
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/private-ips')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public1')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/public1/cassandra')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public2')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/public2/cassandra')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public3')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/public3/cassandra')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/load_balancer')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/taskqueue_master')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/zookeeper')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/db_master')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/taskqueue')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow')
        utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/login')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/memcache')
        utils.should_receive('mkdir').with_args(
            '/tmp/foobaz/symlinks/appengine')

        # fake creation of symlink to for friendly navigation
        links_mapping = {
            '../../public1': [
                '/tmp/foobaz/symlinks/private-ips/private1',
                '/tmp/foobaz/symlinks/load_balancer/public1',
                '/tmp/foobaz/symlinks/taskqueue_master/public1',
                '/tmp/foobaz/symlinks/zookeeper/public1',
                '/tmp/foobaz/symlinks/db_master/public1',
                '/tmp/foobaz/symlinks/taskqueue/public1',
                '/tmp/foobaz/symlinks/shadow/public1',
                '/tmp/foobaz/symlinks/login/public1',
            ],
            '../../public2': [
                '/tmp/foobaz/symlinks/private-ips/private2',
                '/tmp/foobaz/symlinks/zookeeper/public2',
                '/tmp/foobaz/symlinks/appengine/public2',
                '/tmp/foobaz/symlinks/memcache/public2',
            ],
            '../../public3': [
                '/tmp/foobaz/symlinks/private-ips/private3',
                '/tmp/foobaz/symlinks/appengine/public3',
                '/tmp/foobaz/symlinks/memcache/public3',
            ]
        }
        for original_dir, expected_links in links_mapping.iteritems():
            for expected_link in expected_links:
                os.should_receive('symlink').with_args(original_dir,
                                                       expected_link)

        # finally, fake the copying of the log files
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(
            re.compile('/var/log/haproxy*'),
            shell=True,
            stdout=self.fake_temp_file,
            stderr=subprocess.STDOUT).and_return(self.success)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(
            re.compile('/var/log/rabbitmq'),
            shell=True,
            stdout=self.fake_temp_file,
            stderr=subprocess.STDOUT).and_return(self.success)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)
        subprocess.should_receive('Popen').with_args(
            re.compile('/opt/cassandra/cassandra/logs'),
            shell=True,
            stdout=self.fake_temp_file,
            stderr=subprocess.STDOUT).and_return(self.success)

        argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.gather_logs(options)
    def test_terminate_in_virtual_cluster_and_succeeds(self):
        # let's say that there is a locations.yaml file, which means appscale is
        # running, so we should terminate the services on each box
        flexmock(os.path)
        os.path.should_call('exists')  # set up the fall-through
        os.path.should_receive('exists').with_args(
            LocalState.get_secret_key_location(self.keyname)).and_return(True)

        # mock out reading the locations.yaml file, and pretend that we're on
        # a virtualized cluster
        builtins = flexmock(sys.modules['__builtin__'])
        builtins.should_call('open')

        fake_yaml_file = flexmock(name='fake_file')
        fake_yaml_file.should_receive('read').and_return(
            yaml.dump({'infrastructure': 'xen'}))
        builtins.should_receive('open').with_args(
          LocalState.get_locations_yaml_location(self.keyname), 'r') \
          .and_return(fake_yaml_file)

        # mock out reading the json file, and pretend that we're running in a
        # two node deployment
        os.path.should_receive('exists').with_args(
            LocalState.get_locations_json_location(
                self.keyname)).and_return(True)

        fake_json_file = flexmock(name='fake_file')
        fake_json_file.should_receive('read').and_return(
            json.dumps([{
                'public_ip': 'public1',
                'jobs': ['shadow']
            }, {
                'public_ip': 'public2',
                'jobs': ['appengine']
            }]))
        builtins.should_receive('open').with_args(
          LocalState.get_locations_json_location(self.keyname), 'r') \
          .and_return(fake_json_file)

        # and slip in a fake secret file
        fake_secret_file = flexmock(name='fake_file')
        fake_secret_file.should_receive('read').and_return('the secret')
        builtins.should_receive('open').with_args(
          LocalState.get_secret_key_location(self.keyname), 'r') \
          .and_return(fake_secret_file)

        # mock out talking to the appcontroller, and assume that it tells us there
        # there are still two machines in this deployment
        fake_appcontroller = flexmock(name='fake_appcontroller')
        fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
          .and_return(json.dumps(['public1', 'public2']))

        flexmock(SOAPpy)
        SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
          .and_return(fake_appcontroller)

        # and mock out the ssh call to kill the remote appcontroller, assuming that
        # it fails the first time and passes the second
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('controller stop'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.failed).and_return(self.success)

        # next, mock out our checks to see how the stopping process is going and
        # assume that it has stopped
        flexmock(subprocess)
        subprocess.should_receive('Popen').with_args(re.compile('ps x'),
          shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
          .and_return(self.success)

        # finally, mock out removing the yaml file, json file, and secret key from
        # this machine
        flexmock(os)
        os.should_receive('remove').with_args(
            LocalState.get_locations_yaml_location(self.keyname)).and_return()
        os.should_receive('remove').with_args(
            LocalState.get_locations_json_location(self.keyname)).and_return()
        os.should_receive('remove').with_args(
            LocalState.get_secret_key_location(self.keyname)).and_return()

        # also mock out asking the user for confirmation on shutting down
        # their cloud
        builtins.should_receive('raw_input').and_return('yes')

        argv = ["--keyname", self.keyname]
        options = ParseArgs(argv, self.function).args
        AppScaleTools.terminate_instances(options)
Пример #23
0
  def setUp(self):
    # mock out all logging, since it clutters our output
    flexmock(AppScaleLogger)
    AppScaleLogger.should_receive('log').and_return()

    # mock out all sleeps, as they aren't necessary for unit testing
    flexmock(time)
    time.should_receive('sleep').and_return()

    # set up some fake options so that we don't have to generate them via
    # ParseArgs
    self.options = flexmock(infrastructure='ec2', group='boogroup',
      machine='ami-ABCDEFG', instance_type='m1.large', keyname='bookey',
      table='cassandra', verbose=False)
    self.node_layout = NodeLayout(self.options)

    # mock out calls to EC2
    # begin by assuming that our ssh keypair doesn't exist, and thus that we
    # need to create it
    key_contents = "key contents here"
    fake_key = flexmock(name="fake_key", material=key_contents)

    fake_ec2 = flexmock(name="fake_ec2")
    fake_ec2.should_receive('get_key_pair').with_args('bookey') \
      .and_return(None)
    fake_ec2.should_receive('create_key_pair').with_args('bookey') \
      .and_return(fake_key)

    # mock out writing the secret key
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('write').and_return()
    builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    # also, mock out the keypair writing and chmod'ing
    ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.key"
    fake_file = flexmock(name="fake_file")
    fake_file.should_receive('write').with_args(key_contents).and_return()

    builtins.should_receive('open').with_args(ssh_key_location, 'w') \
      .and_return(fake_file)

    flexmock(os)
    os.should_receive('chmod').with_args(ssh_key_location, 0600).and_return()

    # next, assume there are no security groups up yet
    fake_ec2.should_receive('get_all_security_groups').and_return([])

    # and then assume we can create and open our security group fine
    fake_ec2.should_receive('create_security_group').with_args('boogroup',
      'AppScale security group').and_return()
    fake_ec2.should_receive('authorize_security_group').and_return()

    # next, add in mocks for run_instances
    # the first time around, let's say that no machines are running
    # the second time around, let's say that our machine is pending
    # and that it's up the third time around
    fake_pending_instance = flexmock(state='pending')
    fake_pending_reservation = flexmock(instances=fake_pending_instance)

    fake_running_instance = flexmock(state='running', key_name='bookey',
      id='i-12345678', public_dns_name='public1', private_dns_name='private1')
    fake_running_reservation = flexmock(instances=fake_running_instance)

    fake_ec2.should_receive('get_all_instances').and_return([]) \
      .and_return([fake_pending_reservation]) \
      .and_return([fake_running_reservation])

    # next, assume that our run_instances command succeeds
    fake_ec2.should_receive('run_instances').and_return()

    # finally, inject our mocked EC2
    flexmock(boto)
    boto.should_receive('connect_ec2').with_args('baz', 'baz').and_return(fake_ec2)

    # assume that ssh comes up on the third attempt
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      RemoteHelper.SSH_PORT)).and_raise(Exception).and_raise(Exception) \
      .and_return(None)
    flexmock(socket)
    socket.should_receive('socket').and_return(fake_socket)

    # throw some default mocks together for when invoking via shell succeeds
    # and when it fails
    self.fake_temp_file = flexmock(name='fake_temp_file')
    self.fake_temp_file.should_receive('read').and_return('boo out')
    self.fake_temp_file.should_receive('close').and_return()

    flexmock(tempfile)
    tempfile.should_receive('TemporaryFile').and_return(self.fake_temp_file)

    self.success = flexmock(name='success', returncode=0)
    self.success.should_receive('wait').and_return(0)

    self.failed = flexmock(name='success', returncode=1)
    self.failed.should_receive('wait').and_return(1)

    # and assume that we can ssh in as ubuntu to enable root login, but that
    # it fails the first time
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('ubuntu'), \
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed).and_return(self.success)

    # also assume that we can scp over our ssh keys, but that it fails the first
    # time
    subprocess.should_receive('Popen').with_args(re.compile('/root/.ssh/id_'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed).and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile(
      '/root/.appscale/bookey.key'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.failed).and_return(self.success)
  def test_upload_app_successfully(self):
    # add in mocks so that there is an app.yaml, but with no appid set
    flexmock(os.path)
    os.path.should_call('exists')
    app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
    os.path.should_receive('exists').with_args(app_yaml_location) \
      .and_return(True)

    # mock out reading the app.yaml file
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    fake_app_yaml = flexmock(name="fake_app_yaml")
    fake_app_yaml.should_receive('read').and_return(yaml.dump({
      'application' : 'baz',
      'runtime' : 'python'
    }))
    builtins.should_receive('open').with_args(app_yaml_location, 'r') \
      .and_return(fake_app_yaml)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('done_uploading').with_args('baz',
      '/var/apps/baz/app/baz.tar.gz', 'the secret').and_return()
    fake_appcontroller.should_receive('update').with_args(['baz'],
      'the secret').and_return()
    fake_appcontroller.should_receive('is_app_running').with_args('baz',
      'the secret').and_return(False).and_return(True)
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
    fake_userappserver.should_receive('commit_new_user').with_args(
      '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
    fake_userappserver.should_receive('commit_new_user').with_args(
      'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
    fake_userappserver.should_receive('get_app_data').with_args(
      'baz', 'the secret').and_return('\n\nnum_ports:0\n') \
      .and_return(app_data).and_return(app_data).and_return(app_data)
    fake_userappserver.should_receive('commit_new_app').with_args(
      'baz', '*****@*****.**', 'python', 'the secret').and_return('true')
    SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
    flexmock(getpass)
    getpass.should_receive('getpass').and_return('aaaaaa')

    # mock out making the remote app directory
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and mock out tarring and copying the app
    subprocess.should_receive('Popen').with_args(re.compile('tar -czf'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    subprocess.should_receive('Popen').with_args(re.compile(
      '/tmp/appscale-app-baz.tar.gz'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # as well as removing the tar'ed app once we're done copying it
    flexmock(os)
    os.should_receive('remove').with_args('/tmp/appscale-app-baz.tar.gz') \
      .and_return()

    # and slap in a mock that says the app comes up after waiting for it
    # three times
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      8080)).and_raise(Exception).and_raise(Exception) \
      .and_return(None)
    flexmock(socket)
    socket.should_receive('socket').and_return(fake_socket)

    argv = [
      "--keyname", self.keyname,
      "--file", self.app_dir
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.upload_app(options)
Пример #25
0
  def test_add_nodes_in_virt_deployment(self):
    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_yaml = """
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
    """

    # mock out reading the yaml file, and slip in our own
    fake_yaml_file = flexmock(name='fake_yaml')
    fake_yaml_file.should_receive('read').and_return(ips_yaml)
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through
    builtins.should_receive('open').with_args('/tmp/boo.yaml', 'r') \
      .and_return(fake_yaml_file)

    # and pretend we're on a virtualized cluster
    locations_yaml = yaml.dump({
      "infrastructure" : "xen"
    })
    fake_locations_yaml_file = flexmock(name='fake_yaml')
    fake_locations_yaml_file.should_receive('read').and_return(locations_yaml)
    builtins.should_receive('open').with_args(
      LocalState.get_locations_yaml_location(self.keyname), 'r') \
      .and_return(fake_locations_yaml_file)

    # say that the ssh key works
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('ssh'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT,
      stdin=self.fake_input_file) \
      .and_return(self.success)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "public1",
      "private_ip" : "private1",
      "jobs" : ["shadow", "login"]
    }]))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out the SOAP call to the AppController and assume it succeeded
    json_node_info = json.dumps(yaml.safe_load(ips_yaml))
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('start_roles_on_nodes') \
      .with_args(json_node_info, 'the secret').and_return('OK')
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    argv = [
      "--ips", "/tmp/boo.yaml",
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.add_instances(options)
  def test_appscale_with_ips_layout_flag_and_success(self):
    # assume that ssh is running on each machine
    fake_socket = flexmock(name='socket')
    fake_socket.should_receive('connect').with_args(('1.2.3.4', 22)) \
      .and_return(None)
    fake_socket.should_receive('connect').with_args(('1.2.3.5', 22)) \
      .and_return(None)
    fake_socket.should_receive('connect').with_args(('1.2.3.6', 22)) \
      .and_return(None)

    flexmock(socket)
    socket.should_receive('socket').and_return(fake_socket)

    # assume that we have ssh-keygen and ssh-copy-id
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('which ssh-keygen'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('which ssh-copy-id'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # assume that we have a ~/.appscale
    flexmock(os.path)
    os.path.should_call('exists')
    os.path.should_receive('exists').with_args(LocalState.LOCAL_APPSCALE_PATH) \
      .and_return(True)

    # and assume that we don't have public and private keys already made
    path = LocalState.LOCAL_APPSCALE_PATH + self.keyname
    public_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname + '.pub'
    private_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname + '.key'

    os.path.should_receive('exists').with_args(public_key).and_return(False)
    os.path.should_receive('exists').with_args(private_key).and_return(False)

    # next, assume that ssh-keygen ran fine
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('ssh-keygen'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # assume that we can rename the private key
    flexmock(shutil)
    shutil.should_receive('copy').with_args(path, private_key).and_return()

    # finally, assume that we can chmod 0600 those files fine
    flexmock(os)
    os.should_receive('chmod').with_args(public_key, 0600).and_return()
    os.should_receive('chmod').with_args(path, 0600).and_return()

    # and assume that we can ssh-copy-id to each of the three IPs below
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('ssh-copy-id'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # also, we should be able to copy over our new public and private keys fine
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('id_rsa[.pub]?'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.5
appengine: 1.2.3.6
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.add_keypair(options)
  def test_appscale_in_one_node_virt_deployment(self):
    # let's say that appscale isn't already running
    flexmock(os.path)
    os.path.should_call('exists')
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_yaml_location(self.keyname)).and_return(False)

    # mock out talking to logs.appscale.com
    fake_connection = flexmock(name='fake_connection')
    fake_connection.should_receive('request').with_args('POST', '/upload', str,
      AppScaleLogger.HEADERS).and_return()

    flexmock(httplib)
    httplib.should_receive('HTTPSConnection').with_args('logs.appscale.com') \
      .and_return(fake_connection)

    # mock out generating the secret key
    flexmock(uuid)
    uuid.should_receive('uuid4').and_return('the secret')

    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    builtins.should_receive('open').with_args(secret_key_location, 'w') \
      .and_return(fake_secret)

    # mock out seeing if the image is appscale-compatible, and assume it is
    # mock out our attempts to find /etc/appscale and presume it does exist
    flexmock(subprocess)
    subprocess.should_receive('Popen').with_args(re.compile('/etc/appscale'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out our attempts to find /etc/appscale/version and presume it does
    # exist
    subprocess.should_receive('Popen').with_args(re.compile(
      '/etc/appscale/{0}'.format(APPSCALE_VERSION)),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # put in a mock indicating that the database the user wants is supported
    subprocess.should_receive('Popen').with_args(re.compile(
      '/etc/appscale/{0}/{1}'.format(APPSCALE_VERSION, 'cassandra')),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out generating the private key
    flexmock(M2Crypto.RSA)
    fake_rsa_key = flexmock(name='fake_rsa_key')
    fake_rsa_key.should_receive('save_key').with_args(
      LocalState.get_private_key_location(self.keyname), None)
    M2Crypto.RSA.should_receive('gen_key').and_return(fake_rsa_key)

    flexmock(M2Crypto.EVP)
    fake_pkey = flexmock(name='fake_pkey')
    fake_pkey.should_receive('assign_rsa').with_args(fake_rsa_key).and_return()
    M2Crypto.EVP.should_receive('PKey').and_return(fake_pkey)

    # and mock out generating the certificate
    flexmock(M2Crypto.X509)
    fake_cert = flexmock(name='fake_x509')
    fake_cert.should_receive('set_pubkey').with_args(fake_pkey).and_return()
    fake_cert.should_receive('set_subject')
    fake_cert.should_receive('set_issuer_name')
    fake_cert.should_receive('set_not_before')
    fake_cert.should_receive('set_not_after')
    fake_cert.should_receive('sign').with_args(fake_pkey, md="sha256")
    fake_cert.should_receive('save_pem').with_args(
      LocalState.get_certificate_location(self.keyname))
    M2Crypto.X509.should_receive('X509').and_return(fake_cert)

    # assume that we started god fine
    subprocess.should_receive('Popen').with_args(re.compile('god &'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and that we copied over the AppController's god file
    subprocess.should_receive('Popen').with_args(re.compile(
      'appcontroller.god'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # also, that we started the AppController itself
    subprocess.should_receive('Popen').with_args(re.compile('god load'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # assume that the AppController comes up on the third attempt
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('1.2.3.4',
      AppControllerClient.PORT)).and_raise(Exception).and_raise(Exception) \
      .and_return(None)

    # same for the UserAppServer
    fake_socket.should_receive('connect').with_args(('1.2.3.4',
      UserAppClient.PORT)).and_raise(Exception).and_raise(Exception) \
      .and_return(None)

    # as well as for the AppLoadBalancer
    fake_socket.should_receive('connect').with_args(('1.2.3.4',
      RemoteHelper.APP_LOAD_BALANCER_PORT)).and_raise(Exception) \
      .and_raise(Exception).and_return(None)

    flexmock(socket)
    socket.should_receive('socket').and_return(fake_socket)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('set_parameters').with_args(list, list,
      ['none'], 'the secret').and_return('OK')
    fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['1.2.3.4']))
    role_info = [{
      'public_ip' : '1.2.3.4',
      'private_ip' : '1.2.3.4',
      'jobs' : ['shadow', 'login']
    }]
    fake_appcontroller.should_receive('get_role_info').with_args('the secret') \
      .and_return(json.dumps(role_info))
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('nothing interesting here') \
      .and_return('Database is at not-up-yet') \
      .and_return('Database is at 1.2.3.4')
    fake_appcontroller.should_receive('is_done_initializing') \
      .and_raise(Exception) \
      .and_return(False) \
      .and_return(True)
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps([{
      "public_ip" : "1.2.3.4",
      "private_ip" : "1.2.3.4",
      "jobs" : ["shadow", "login"]
    }]))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'w') \
      .and_return(fake_nodes_json)

    # copying over the locations yaml and json files should be fine
    subprocess.should_receive('Popen').with_args(re.compile(
      'locations-{0}.[yaml|json]'.format(self.keyname)),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # same for the secret key
    subprocess.should_receive('Popen').with_args(re.compile(
      '{0}.secret'.format(self.keyname)),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # mock out calls to the UserAppServer and presume that calls to create new
    # users succeed
    fake_userappserver = flexmock(name='fake_appcontroller')
    fake_userappserver.should_receive('commit_new_user').with_args(
      '*****@*****.**', str, 'xmpp_user', 'the secret') \
      .and_return('true')
    fake_userappserver.should_receive('commit_new_user').with_args(
      '[email protected]', str, 'xmpp_user', 'the secret') \
      .and_return('true')
    fake_userappserver.should_receive('set_cloud_admin_status').with_args(
      '*****@*****.**', 'true', 'the secret').and_return()
    fake_userappserver.should_receive('set_capabilities').with_args(
      '*****@*****.**', UserAppClient.ADMIN_CAPABILITIES, 'the secret').and_return()
    SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:4343') \
      .and_return(fake_userappserver)

    # don't use a 192.168.X.Y IP here, since sometimes we set our virtual
    # machines to boot with those addresses (and that can mess up our tests).
    ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine:  1.2.3.4
    """)

    argv = [
      "--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
      "--keyname", self.keyname,
      "--test"
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.run_instances(options)