def terminate_instances(cls, options): """Stops all services running in an AppScale deployment, and in cloud deployments, also powers off the instances previously spawned. Raises: AppScaleException: If AppScale is not running, and thus can't be terminated. """ if not os.path.exists( LocalState.get_secret_key_location(options.keyname)): raise AppScaleException( "AppScale is not running with the keyname {0}".format( options.keyname)) infrastructure = LocalState.get_infrastructure(options.keyname) # If the user is on a cloud deployment, and not backing their data to # persistent disks, warn them before shutting down AppScale. # Also, if we're in developer mode, skip the warning. if infrastructure != "xen" and not LocalState.are_disks_used( options.keyname) and not options.test: LocalState.ensure_user_wants_to_terminate() if infrastructure in InfrastructureAgentFactory.VALID_AGENTS: RemoteHelper.terminate_cloud_infrastructure( options.keyname, options.verbose) else: RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose) LocalState.cleanup_appscale_files(options.keyname) AppScaleLogger.success( "Successfully shut down your AppScale deployment.")
def terminate_instances(cls, options): """Stops all services running in an AppScale deployment, and in cloud deployments, also powers off the instances previously spawned. Raises: AppScaleException: If AppScale is not running, and thus can't be terminated. """ if not os.path.exists(LocalState.get_secret_key_location(options.keyname)): raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname)) infrastructure = LocalState.get_infrastructure(options.keyname) # If the user is on a cloud deployment, and not backing their data to # persistent disks, warn them before shutting down AppScale. # Also, if we're in developer mode, skip the warning. if infrastructure != "xen" and not LocalState.are_disks_used(options.keyname) and not options.test: LocalState.ensure_user_wants_to_terminate() if infrastructure in InfrastructureAgentFactory.VALID_AGENTS: RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose) else: RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose) LocalState.cleanup_appscale_files(options.keyname) AppScaleLogger.success("Successfully shut down your AppScale deployment.")
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args('/tmp/foobaz').and_return( False) # and mock out the mkdir operation flexmock(os) os.should_receive('mkdir').with_args('/tmp/foobaz').and_return() # next, mock out finding the login ip address os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps([{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }])) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args( 'the secret').and_return(json.dumps(['public1', 'public2'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # fake the creation of the log directories locally os.should_receive('mkdir').with_args( '/tmp/foobaz/public1').and_return() os.should_receive('mkdir').with_args( '/tmp/foobaz/public2').and_return() # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)
def shut_down_appscale_if_running(cls, options): """ Checks if AppScale is running and shuts it down as this is an offline upgrade. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if os.path.exists(LocalState.get_secret_key_location(options.keyname)): AppScaleLogger.warn("AppScale needs to be down for this upgrade. " "Upgrade process could take a while and it is not reversible.") if not options.test: response = raw_input( 'Are you sure you want to proceed with shutting down AppScale to ' 'continue the upgrade? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled AppScale upgrade.") AppScaleLogger.log("Shutting down AppScale...") cls.terminate_instances(options) else: AppScaleLogger.warn("Upgrade process could take a while and it is not reversible.") if options.test: return response = raw_input( 'Are you sure you want to proceed with the upgrade? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled AppScale upgrade.")
def copy_deployment_credentials(cls, host, options): """Copies credentials needed to start the AppController and have it create other instances (in cloud deployments). Args: host: A str representing the machine (reachable from this computer) to copy our deployment credentials to. options: A Namespace that indicates which SSH keypair to use, and whether or not we are running in a cloud infrastructure. """ cls.scp(host, options.keyname, LocalState.get_secret_key_location( options.keyname), '/etc/appscale/secret.key', options.verbose) cls.scp(host, options.keyname, LocalState.get_key_path_from_name( options.keyname), '/etc/appscale/ssh.key', options.verbose) LocalState.generate_ssl_cert(options.keyname, options.verbose) cls.scp(host, options.keyname, LocalState.get_certificate_location( options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose) cls.scp(host, options.keyname, LocalState.get_private_key_location( options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose) AppScaleLogger.log("Copying over deployment credentials") cert = LocalState.get_certificate_location(options.keyname) private_key = LocalState.get_private_key_location(options.keyname) cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1', options.verbose) cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem", options.verbose) cls.scp(host, options.keyname, private_key, "/etc/appscale/keys/cloud1/mykey.pem", options.verbose)
def shut_down_appscale_if_running(cls, options): """ Checks if AppScale is running and shuts it down as this is an offline upgrade. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if os.path.exists(LocalState.get_secret_key_location(options.keyname)): AppScaleLogger.warn( "AppScale needs to be down for this upgrade. " "Upgrade process could take a while and it is not reversible." ) if not options.test: response = raw_input( "Are you sure you want to proceed with shutting down AppScale to " "continue the upgrade? (y/N) " ) if response.lower() not in ["y", "yes"]: raise AppScaleException("Cancelled AppScale upgrade.") AppScaleLogger.log("Shutting down AppScale...") cls.terminate_instances(options) else: AppScaleLogger.warn("Upgrade process could take a while and it is not reversible.") if options.test: return response = raw_input("Are you sure you want to proceed with the upgrade? (y/N) ") if response.lower() not in ["y", "yes"]: raise AppScaleException("Cancelled AppScale upgrade.")
def terminate_instances(cls, options): """Stops all services running in an AppScale deployment, and in cloud deployments, also powers off the instances previously spawned. Raises: AppScaleException: If AppScale is not running, and thus can't be terminated. """ try: infrastructure = LocalState.get_infrastructure(options.keyname) except IOError: raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".format(options.keyname)) if infrastructure == "xen" and options.terminate: raise AppScaleException("Terminate option is invalid for cluster mode.") if infrastructure == "xen" or not options.terminate: # We are in cluster mode: let's check if AppScale is running. if not os.path.exists(LocalState.get_secret_key_location(options.keyname)): raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname)) # Stop gracefully the AppScale deployment. try: RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose) except (IOError, AppScaleException): # Don't fail if we cannot find the configuration. pass # And if we are on a cloud infrastructure, terminate instances if # asked. if infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate: RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
def copy_local_metadata(cls, host, keyname, is_verbose): """Copies the locations.yaml and locations.json files found locally (which contain metadata about this AppScale deployment) to the specified host. Args: host: The machine that we should copy the metadata files to. keyname: The name of the SSH keypair that we can use to log into the given host. is_verbose: A bool that indicates if we should print the SCP commands we exec to stdout. """ # copy the metadata files for AppScale itself to use cls.scp(host, keyname, LocalState.get_locations_yaml_location(keyname), '/etc/appscale/locations-{0}.yaml'.format(keyname), is_verbose) cls.scp(host, keyname, LocalState.get_locations_json_location(keyname), '/etc/appscale/locations-{0}.json'.format(keyname), is_verbose) # and copy the json file if the tools on that box wants to use it cls.scp(host, keyname, LocalState.get_locations_json_location(keyname), '/root/.appscale/locations-{0}.json'.format(keyname), is_verbose) # and copy the secret file if the tools on that box wants to use it cls.scp(host, keyname, LocalState.get_secret_key_location(keyname), '/root/.appscale/', is_verbose)
def copy_deployment_credentials(cls, host, options): """Copies credentials needed to start the AppController and have it create other instances (in cloud deployments). Args: host: A str representing the machine (reachable from this computer) to copy our deployment credentials to. options: A Namespace that indicates which SSH keypair to use, and whether or not we are running in a cloud infrastructure. """ cls.scp(host, options.keyname, LocalState.get_secret_key_location(options.keyname), '/etc/appscale/secret.key', options.verbose) cls.scp(host, options.keyname, LocalState.get_key_path_from_name(options.keyname), '/etc/appscale/ssh.key', options.verbose) LocalState.generate_ssl_cert(options.keyname, options.verbose) cls.scp(host, options.keyname, LocalState.get_certificate_location(options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose) cls.scp(host, options.keyname, LocalState.get_private_key_location(options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose) hash_id = subprocess.Popen([ "openssl", "x509", "-hash", "-noout", "-in", LocalState.get_certificate_location(options.keyname) ], stdout=subprocess.PIPE).communicate()[0] cls.ssh(host, options.keyname, 'ln -fs /etc/appscale/certs/mycert.pem /etc/ssl/certs/{0}.0'.\ format(hash_id.rstrip()), options.verbose) AppScaleLogger.log("Copying over deployment credentials") cert = LocalState.get_certificate_location(options.keyname) private_key = LocalState.get_private_key_location(options.keyname) cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1', options.verbose) cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem", options.verbose) cls.scp(host, options.keyname, private_key, "/etc/appscale/keys/cloud1/mykey.pem", options.verbose) # In Google Compute Engine, we also need to copy over our client_secrets # file and the OAuth2 file that the user has approved for use with their # credentials, otherwise the AppScale VMs won't be able to interact with # GCE. if options.infrastructure and options.infrastructure == 'gce': if os.path.exists(LocalState.get_client_secrets_location( \ options.keyname)): cls.scp( host, options.keyname, LocalState.get_client_secrets_location(options.keyname), '/etc/appscale/client_secrets.json', options.verbose) cls.scp(host, options.keyname, LocalState.get_oauth2_storage_location(options.keyname), '/etc/appscale/oauth2.dat', options.verbose)
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False) # and mock out the mkdir operation flexmock(os) os.should_receive('mkdir').with_args('/tmp/foobaz').and_return() # next, mock out finding the login ip address os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }])) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args( 'the secret').and_return(json.dumps(['public1', 'public2'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # fake the creation of the log directories locally os.should_receive('mkdir').with_args('/tmp/foobaz/public1').and_return() os.should_receive('mkdir').with_args('/tmp/foobaz/public2').and_return() # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) argv = [ "--keyname", self.keyname, "--location", "/tmp/foobaz" ] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)
def copy_deployment_credentials(cls, host, options): """Copies credentials needed to start the AppController and have it create other instances (in cloud deployments). Args: host: A str representing the machine (reachable from this computer) to copy our deployment credentials to. options: A Namespace that indicates which SSH keypair to use, and whether or not we are running in a cloud infrastructure. """ local_secret_key = LocalState.get_secret_key_location(options.keyname) cls.scp(host, options.keyname, local_secret_key, '{}/secret.key'.format(cls.CONFIG_DIR), options.verbose) local_ssh_key = LocalState.get_key_path_from_name(options.keyname) cls.scp(host, options.keyname, local_ssh_key, '{}/ssh.key'.format(cls.CONFIG_DIR), options.verbose) LocalState.generate_ssl_cert(options.keyname, options.verbose) local_cert = LocalState.get_certificate_location(options.keyname) cls.scp(host, options.keyname, local_cert, '{}/certs/mycert.pem'.format(cls.CONFIG_DIR), options.verbose) local_private_key = LocalState.get_private_key_location( options.keyname) cls.scp(host, options.keyname, local_private_key, '{}/certs/mykey.pem'.format(cls.CONFIG_DIR), options.verbose) hash_id = subprocess.Popen([ "openssl", "x509", "-hash", "-noout", "-in", LocalState.get_certificate_location(options.keyname) ], stdout=subprocess.PIPE).communicate()[0] symlink_cert = 'ln -fs {}/certs/mycert.pem /etc/ssl/certs/{}.0'.\ format(cls.CONFIG_DIR, hash_id.rstrip()) cls.ssh(host, options.keyname, symlink_cert, options.verbose) # In Google Compute Engine, we also need to copy over our client_secrets # file and the OAuth2 file that the user has approved for use with their # credentials, otherwise the AppScale VMs won't be able to interact with # GCE. if options.infrastructure and options.infrastructure == 'gce': secrets_location = LocalState.get_client_secrets_location( options.keyname) if not os.path.exists(secrets_location): raise AppScaleException( '{} does not exist.'.format(secrets_location)) secrets_type = GCEAgent.get_secrets_type(secrets_location) cls.scp(host, options.keyname, secrets_location, '{}/client_secrets.json'.format(cls.CONFIG_DIR), options.verbose) if secrets_type == CredentialTypes.OAUTH: local_oauth = LocalState.get_oauth2_storage_location( options.keyname) cls.scp(host, options.keyname, local_oauth, '{}/oauth2.dat'.format(cls.CONFIG_DIR), options.verbose)
def test_ensure_appscale_isnt_running_but_it_is(self): # if there is a locations.yaml file and force isn't set, # we should abort os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) self.assertRaises(BadConfigurationException, LocalState.ensure_appscale_isnt_running, self.keyname, False)
def test_ensure_appscale_isnt_running_but_it_is(self): # if there is a locations.yaml file and force isn't set, # we should abort os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) self.assertRaises(BadConfigurationException, LocalState.ensure_appscale_isnt_running, self.keyname, False)
def test_describe_instances_with_two_nodes(self): # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps([ { "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }, { "public_ip": "public2", "private_ip": "private2", "jobs": ["appengine"] }, ])) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # assume that there are two machines running in our deployment argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.describe_instances(options)
def test_remove_app_and_app_is_running(self): # mock out reading from stdin, and assume the user says 'YES' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('YES') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('stop_app').with_args('blargapp', 'the secret').and_return('OK') fake_appcontroller.should_receive('is_app_running').with_args('blargapp', 'the secret').and_return(True).and_return(True).and_return(False) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }])) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out calls to the UserAppServer and presume that the app does exist fake_userappserver = flexmock(name='fake_uaserver') fake_userappserver.should_receive('get_app_data').with_args( 'blargapp', 'the secret').and_return(json.dumps({ 'hosts' : { '192.168.1.1' : { 'http' : '80', 'https' : '443' }}})) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \ .and_return(fake_userappserver) argv = [ "--appname", "blargapp", "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.remove_app(options)
def test_remove_app_and_app_is_running(self): # mock out reading from stdin, and assume the user says 'YES' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('YES') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('stop_app').with_args( 'blargapp', 'the secret').and_return('OK') fake_appcontroller.should_receive('is_app_running').with_args( 'blargapp', 'the secret').and_return(True).and_return(True).and_return(False) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps([{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }])) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out calls to the UserAppServer and presume that the app does exist fake_userappserver = flexmock(name='fake_uaserver') fake_userappserver.should_receive('get_app_data').with_args( 'blargapp', 'the secret').and_return('\nnum_ports:2\n') SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \ .and_return(fake_userappserver) argv = ["--appname", "blargapp", "--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.remove_app(options)
def test_reset_password_for_user_that_doesnt_exist(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the username and new password from the user builtins.should_receive('raw_input').and_return('*****@*****.**') flexmock(getpass) getpass.should_receive('getpass').and_return('the password') # mock out finding the login node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return( json.dumps([{ 'public_ip': 'public1', 'private_ip': 'private1', 'jobs': ['login', 'db_master'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at public1') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str, 'the secret').and_return('Error: user does not exist') SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \ .and_return(fake_userappserver) argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args self.assertRaises(SystemExit, AppScaleTools.reset_password, options)
def test_update_local_metadata(self): # mock out getting all the ips in the deployment from the head node fake_soap = flexmock(name='fake_soap') fake_soap.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1'])) role_info = [{ 'public_ip' : 'public1', 'private_ip' : 'private1', 'jobs' : ['shadow', 'db_master'] }] fake_soap.should_receive('get_role_info').with_args('the secret') \ .and_return(json.dumps(role_info)) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_soap) # mock out reading the secret key fake_secret = flexmock(name='fake_secret') fake_secret.should_receive('read').and_return('the secret') builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_secret_key_location('booscale'), 'r') \ .and_return(fake_secret) # mock out writing the yaml file fake_locations_yaml = flexmock(name='fake_locations_yaml') fake_locations_yaml.should_receive('write').with_args(yaml.dump({ 'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG', 'secret': 'the secret', 'infrastructure': 'ec2', 'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra', 'db_master': 'node-0', 'zone' : 'my-zone-1b' })).and_return() builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location('booscale'), 'w') \ .and_return(fake_locations_yaml) # and mock out writing the json file fake_locations_json = flexmock(name='fake_locations_json') fake_locations_json.should_receive('write').with_args(json.dumps( role_info)).and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location('booscale'), 'w') \ .and_return(fake_locations_json) options = flexmock(name='options', table='cassandra', infrastructure='ec2', keyname='booscale', group='boogroup', zone='my-zone-1b') node_layout = NodeLayout(options={ 'min' : 1, 'max' : 1, 'infrastructure' : 'ec2', 'table' : 'cassandra' }) host = 'public1' instance_id = 'i-ABCDEFG' LocalState.update_local_metadata(options, node_layout, host, instance_id)
def test_update_local_metadata(self): # mock out getting all the ips in the deployment from the head node fake_soap = flexmock(name='fake_soap') fake_soap.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1'])) role_info = [{ 'public_ip' : 'public1', 'private_ip' : 'private1', 'jobs' : ['shadow', 'db_master'] }] fake_soap.should_receive('get_role_info').with_args('the secret') \ .and_return(json.dumps(role_info)) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_soap) # mock out reading the secret key fake_secret = flexmock(name='fake_secret') fake_secret.should_receive('read').and_return('the secret') builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_secret_key_location('booscale'), 'r') \ .and_return(fake_secret) # mock out writing the yaml file fake_locations_yaml = flexmock(name='fake_locations_yaml') fake_locations_yaml.should_receive('write').with_args(yaml.dump({ 'load_balancer': 'public1', 'instance_id': 'i-ABCDEFG', 'secret': 'the secret', 'infrastructure': 'ec2', 'group': 'boogroup', 'ips': 'public1', 'table': 'cassandra', 'db_master': 'node-0', 'zone' : 'my-zone-1b' })).and_return() builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location('booscale'), 'w') \ .and_return(fake_locations_yaml) # and mock out writing the json file fake_locations_json = flexmock(name='fake_locations_json') fake_locations_json.should_receive('write').with_args(json.dumps( role_info)).and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location('booscale'), 'w') \ .and_return(fake_locations_json) options = flexmock(name='options', table='cassandra', infrastructure='ec2', keyname='booscale', group='boogroup', zone='my-zone-1b') node_layout = NodeLayout(options={ 'min' : 1, 'max' : 1, 'infrastructure' : 'ec2', 'table' : 'cassandra' }) host = 'public1' instance_id = 'i-ABCDEFG' LocalState.update_local_metadata(options, node_layout, host, instance_id)
def test_describe_instances_with_two_nodes(self): # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }, { "public_ip" : "public2", "private_ip" : "private2", "jobs" : ["appengine"] }, ])) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # assume that there are two machines running in our deployment argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.describe_instances(options)
def test_terminate_when_not_running(self): # let's say that there's no locations.yaml file, which means appscale isn't # running, so we should throw up and die flexmock(os.path) os.path.should_call("exists") # set up the fall-through os.path.should_receive("exists").with_args(LocalState.get_secret_key_location(self.keyname)).and_return(False) argv = ["--keyname", self.keyname, "--test"] options = ParseArgs(argv, self.function).args self.assertRaises(AppScaleException, AppScaleTools.terminate_instances, options)
def test_reset_password_for_user_that_exists(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the username and new password from the user builtins.should_receive('raw_input').and_return('*****@*****.**') flexmock(getpass) getpass.should_receive('getpass').and_return('the password') # mock out finding the login node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return(json.dumps([{ 'public_ip' : 'public1', 'private_ip' : 'private1', 'jobs' : ['login', 'db_master'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at public1') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', str, 'the secret').and_return('true') SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \ .and_return(fake_userappserver) argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.reset_password(options)
def test_terminate_when_not_running(self): # let's say that there's no locations.yaml file, which means appscale isn't # running, so we should throw up and die flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(False) argv = ["--keyname", self.keyname, "--test"] options = ParseArgs(argv, self.function).args self.assertRaises(AppScaleException, AppScaleTools.terminate_instances, options)
def copy_deployment_credentials(cls, host, options): """Copies credentials needed to start the AppController and have it create other instances (in cloud deployments). Args: host: A str representing the machine (reachable from this computer) to copy our deployment credentials to. options: A Namespace that indicates which SSH keypair to use, and whether or not we are running in a cloud infrastructure. """ cls.scp(host, options.keyname, LocalState.get_secret_key_location( options.keyname), '/etc/appscale/secret.key', options.verbose) cls.scp(host, options.keyname, LocalState.get_key_path_from_name( options.keyname), '/etc/appscale/ssh.key', options.verbose) LocalState.generate_ssl_cert(options.keyname, options.verbose) cls.scp(host, options.keyname, LocalState.get_certificate_location( options.keyname), '/etc/appscale/certs/mycert.pem', options.verbose) cls.scp(host, options.keyname, LocalState.get_private_key_location( options.keyname), '/etc/appscale/certs/mykey.pem', options.verbose) hash_id = subprocess.Popen(["openssl", "x509", "-hash", "-noout", "-in", LocalState.get_certificate_location(options.keyname)], stdout=subprocess.PIPE).communicate()[0] cls.ssh(host, options.keyname, 'ln -fs /etc/appscale/certs/mycert.pem /etc/ssl/certs/{0}.0'.\ format(hash_id.rstrip()), options.verbose) AppScaleLogger.log("Copying over deployment credentials") cert = LocalState.get_certificate_location(options.keyname) private_key = LocalState.get_private_key_location(options.keyname) cls.ssh(host, options.keyname, 'mkdir -p /etc/appscale/keys/cloud1', options.verbose) cls.scp(host, options.keyname, cert, "/etc/appscale/keys/cloud1/mycert.pem", options.verbose) cls.scp(host, options.keyname, private_key, "/etc/appscale/keys/cloud1/mykey.pem", options.verbose) # In Google Compute Engine, we also need to copy over our client_secrets # file and the OAuth2 file that the user has approved for use with their # credentials, otherwise the AppScale VMs won't be able to interact with # GCE. if options.infrastructure and options.infrastructure == 'gce': if os.path.exists(LocalState.get_client_secrets_location( \ options.keyname)): cls.scp(host, options.keyname, LocalState.get_client_secrets_location( options.keyname), '/etc/appscale/client_secrets.json', options.verbose) cls.scp(host, options.keyname, LocalState.get_oauth2_storage_location( options.keyname), '/etc/appscale/oauth2.dat', options.verbose)
def copy_deployment_credentials(cls, host, options): """Copies credentials needed to start the AppController and have it create other instances (in cloud deployments). Args: host: A str representing the machine (reachable from this computer) to copy our deployment credentials to. options: A Namespace that indicates which SSH keypair to use, and whether or not we are running in a cloud infrastructure. """ local_secret_key = LocalState.get_secret_key_location(options.keyname) cls.scp(host, options.keyname, local_secret_key, '{}/secret.key'.format(cls.CONFIG_DIR), options.verbose) local_ssh_key = LocalState.get_key_path_from_name(options.keyname) cls.scp(host, options.keyname, local_ssh_key, '{}/ssh.key'.format(cls.CONFIG_DIR), options.verbose) LocalState.generate_ssl_cert(options.keyname, options.verbose) local_cert = LocalState.get_certificate_location(options.keyname) cls.scp(host, options.keyname, local_cert, '{}/certs/mycert.pem'.format(cls.CONFIG_DIR), options.verbose) local_private_key = LocalState.get_private_key_location(options.keyname) cls.scp(host, options.keyname, local_private_key, '{}/certs/mykey.pem'.format(cls.CONFIG_DIR), options.verbose) hash_id = subprocess.Popen(["openssl", "x509", "-hash", "-noout", "-in", LocalState.get_certificate_location(options.keyname)], stdout=subprocess.PIPE).communicate()[0] symlink_cert = 'ln -fs {}/certs/mycert.pem /etc/ssl/certs/{}.0'.\ format(cls.CONFIG_DIR, hash_id.rstrip()) cls.ssh(host, options.keyname, symlink_cert, options.verbose) # In Google Compute Engine, we also need to copy over our client_secrets # file and the OAuth2 file that the user has approved for use with their # credentials, otherwise the AppScale VMs won't be able to interact with # GCE. if options.infrastructure and options.infrastructure == 'gce': secrets_location = LocalState.get_client_secrets_location(options.keyname) if not os.path.exists(secrets_location): raise AppScaleException('{} does not exist.'.format(secrets_location)) secrets_type = GCEAgent.get_secrets_type(secrets_location) cls.scp(host, options.keyname, secrets_location, '{}/client_secrets.json'.format(cls.CONFIG_DIR), options.verbose) if secrets_type == CredentialTypes.OAUTH: local_oauth = LocalState.get_oauth2_storage_location(options.keyname) cls.scp(host, options.keyname, local_oauth, '{}/oauth2.dat'.format(cls.CONFIG_DIR), options.verbose)
def test_get_property(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out finding the shadow node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return( json.dumps([{ 'public_ip': 'public1', 'private_ip': 'private1', 'jobs': ['login', 'shadow'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller property_name = "name" property_value = "value" fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_property').with_args( property_name, property_value, 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = [ "--keyname", self.keyname, "--property_name", property_name, "--property_value", property_value ] options = ParseArgs(argv, self.function).args result = AppScaleTools.set_property(options) self.assertEqual(None, result)
def terminate_cloud_instance(cls, instance_id, options): """Powers off a single instance in the currently AppScale deployment and cleans up secret key from the local filesystem. Args: instance_id: str containing the instance id. options: namespace containing the run parameters. """ AppScaleLogger.log("About to terminate instance {0}".format(instance_id)) agent = InfrastructureAgentFactory.create_agent(options.infrastructure) params = agent.get_params_from_args(options) params["IS_VERBOSE"] = options.verbose params[agent.PARAM_INSTANCE_IDS] = [instance_id] agent.terminate_instances(params) agent.cleanup_state(params) os.remove(LocalState.get_secret_key_location(options.keyname))
def test_remove_app_but_app_isnt_running(self): # mock out reading from stdin, and assume the user says 'yes' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('yes') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('does_app_exist').with_args('blargapp', 'the secret').and_return(False) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }])) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) argv = [ "--appname", "blargapp", "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args self.assertRaises(AppScaleException, AppScaleTools.remove_app, options)
def test_get_property(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out finding the shadow node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return(json.dumps([{ 'public_ip' : 'public1', 'private_ip' : 'private1', 'jobs' : ['login', 'shadow'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller property_name = "name" property_value = "value" fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_property').with_args(property_name, property_value, 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = [ "--keyname", self.keyname, "--property_name", property_name, "--property_value", property_value ] options = ParseArgs(argv, self.function).args result = AppScaleTools.set_property(options) self.assertEqual(None, result)
def terminate_cloud_instance(cls, instance_id, options): """Powers off a single instance in the currently AppScale deployment and cleans up secret key from the local filesystem. Args: instance_id: str containing the instance id. options: namespace containing the run parameters. """ AppScaleLogger.log("About to terminate instance {0}" .format(instance_id)) agent = InfrastructureAgentFactory.create_agent(options.infrastructure) params = agent.get_params_from_args(options) params['IS_VERBOSE'] = options.verbose params[agent.PARAM_INSTANCE_IDS] = [instance_id] agent.terminate_instances(params) agent.cleanup_state(params) os.remove(LocalState.get_secret_key_location(options.keyname))
def copy_local_metadata(cls, host, keyname, is_verbose): """Copies the locations.json file found locally (which contain metadata about this AppScale deployment) to the specified host. Args: host: The machine that we should copy the metadata files to. keyname: The name of the SSH keypair that we can use to log into the given host. is_verbose: A bool that indicates if we should print the SCP commands we exec to stdout. """ # and copy the json file if the tools on that box wants to use it cls.scp(host, keyname, LocalState.get_locations_json_location(keyname), '{}/locations-{}.json'.format(cls.CONFIG_DIR, keyname), is_verbose) # and copy the secret file if the tools on that box wants to use it cls.scp(host, keyname, LocalState.get_secret_key_location(keyname), cls.CONFIG_DIR, is_verbose)
def terminate_instances(cls, options): """Stops all services running in an AppScale deployment, and in cloud deployments, also powers off the instances previously spawned. Raises: AppScaleException: If AppScale is not running, and thus can't be terminated. """ try: infrastructure = LocalState.get_infrastructure(options.keyname) except IOError: raise AppScaleException( "Cannot find AppScale's configuration for keyname {0}".format( options.keyname)) if infrastructure == "xen" and options.terminate: raise AppScaleException( "Terminate option is invalid for cluster mode.") if infrastructure == "xen" or not options.terminate: # We are in cluster mode: let's check if AppScale is running. if not os.path.exists( LocalState.get_secret_key_location(options.keyname)): raise AppScaleException( "AppScale is not running with the keyname {0}".format( options.keyname)) # Stop gracefully the AppScale deployment. try: RemoteHelper.terminate_virtualized_cluster(options.keyname, options.clean, options.verbose) except (IOError, AppScaleException): # Don't fail if we cannot find the configuration. pass # And if we are on a cloud infrastructure, terminate instances if # asked. if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate): RemoteHelper.terminate_cloud_infrastructure( options.keyname, options.verbose) if options.clean: LocalState.clean_local_metadata(keyname=options.keyname)
def setUp(self): self.keyname = "boobazblargfoo" self.function = "appscale-relocate-app" self.appid = 'my-crazy-app' # mock out any writing to stdout flexmock(AppScaleLogger) AppScaleLogger.should_receive('log').and_return() AppScaleLogger.should_receive('success').and_return() AppScaleLogger.should_receive('warn').and_return() # mock out all sleeping flexmock(time) time.should_receive('sleep').and_return() # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps([{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }])) fake_nodes_json.should_receive('write').and_return() builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # put in a mock for reading the secret file secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret)
def setUp(self): self.keyname = "boobazblargfoo" self.function = "appscale-relocate-app" self.appid = 'my-crazy-app' # mock out any writing to stdout flexmock(AppScaleLogger) AppScaleLogger.should_receive('log').and_return() AppScaleLogger.should_receive('success').and_return() AppScaleLogger.should_receive('warn').and_return() # mock out all sleeping flexmock(time) time.should_receive('sleep').and_return() # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }])) fake_nodes_json.should_receive('write').and_return() builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # put in a mock for reading the secret file builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret)
def copy_local_metadata(cls, host, keyname, is_verbose): """Copies the locations.yaml and locations.json files found locally (which contain metadata about this AppScale deployment) to the specified host. Args: host: The machine that we should copy the metadata files to. keyname: The name of the SSH keypair that we can use to log into the given host. is_verbose: A bool that indicates if we should print the SCP commands we exec to stdout. """ # copy the metadata files for AppScale itself to use cls.scp( host, keyname, LocalState.get_locations_yaml_location(keyname), "/etc/appscale/locations-{0}.yaml".format(keyname), is_verbose, ) cls.scp( host, keyname, LocalState.get_locations_json_location(keyname), "/etc/appscale/locations-{0}.json".format(keyname), is_verbose, ) # and copy the json file if the tools on that box wants to use it cls.scp( host, keyname, LocalState.get_locations_json_location(keyname), "/root/.appscale/locations-{0}.json".format(keyname), is_verbose, ) # and copy the secret file if the tools on that box wants to use it cls.scp(host, keyname, LocalState.get_secret_key_location(keyname), "/root/.appscale/", is_verbose)
def test_appscale_in_one_node_virt_deployment(self): # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() rh = flexmock(RemoteHelper) rh.should_receive('copy_deployment_credentials').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out copying over the keys self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.key'),False,5) self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell')\ .with_args(re.compile('^openssl'),False,stdin=None)\ .and_return() # mock out removing the old json file self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\ .and_return() # assume that we started monit fine self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\ .and_return() # and that we copied over the AppController's monit file self.local_state.should_receive('shell')\ .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\ .and_return() self.setup_socket_mocks('1.2.3.4') self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "1.2.3.4", "private_ip" : "1.2.3.4", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.yaml'),\ False,5)\ .and_return() self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.json'),\ False,5)\ .and_return() self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/root/.appscale/locations-bookey.json'),\ False,5)\ .and_return() # same for the secret key self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.secret'),False,5)\ .and_return() self.setup_uaserver_mocks('1.2.3.4') # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment(self): # let's say that appscale isn't already running local_state = flexmock(LocalState) local_state.should_receive('ensure_appscale_isnt_running').and_return() local_state.should_receive('make_appscale_directory').and_return() rh = flexmock(RemoteHelper) rh.should_receive('copy_deployment_credentials').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out copying over the keys local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.key'),False,5) # mock out our attempts to find /etc/appscale and presume it does exist local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale'))\ .and_return() # mock out our attempts to find /etc/appscale/version and presume it does # exist local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale/{0}'.format(APPSCALE_VERSION)))\ .and_return() # finally, put in a mock indicating that the database the user wants # is supported local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale/{0}/{1}'\ .format(APPSCALE_VERSION, 'cassandra')))\ .and_return() # mock out generating the private key local_state.should_receive('shell')\ .with_args(re.compile('^openssl'),False,stdin=None)\ .and_return() # mock out removing the old json file local_state = flexmock(LocalState) local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\ .and_return() # assume that we started god fine local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('god &'))\ .and_return() # and that we copied over the AppController's god file local_state.should_receive('shell')\ .with_args(re.compile('scp .*appcontroller\.god.*'),False,5)\ .and_return() # also, that we started the AppController itself local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('^god load .*appcontroller\.god'))\ .and_return() # assume that the AppController comes up on the third attempt fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('1.2.3.4', AppControllerClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # same for the UserAppServer fake_socket.should_receive('connect').with_args(('1.2.3.4', UserAppClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # as well as for the AppDashboard fake_socket.should_receive('connect').with_args(('1.2.3.4', RemoteHelper.APP_DASHBOARD_PORT)).and_raise(Exception) \ .and_raise(Exception).and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_parameters').with_args(list, list, ['none'], 'the secret').and_return('OK') fake_appcontroller.should_receive('get_all_public_ips')\ .with_args('the secret') \ .and_return(json.dumps(['1.2.3.4'])) role_info = [{ 'public_ip' : '1.2.3.4', 'private_ip' : '1.2.3.4', 'jobs' : ['shadow', 'login'] }] fake_appcontroller.should_receive('get_role_info').with_args('the secret') \ .and_return(json.dumps(role_info)) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') fake_appcontroller.should_receive('is_done_initializing') \ .and_return(False) \ .and_return(True) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "1.2.3.4", "private_ip" : "1.2.3.4", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.yaml'),\ False,5)\ .and_return() local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.json'),\ False,5)\ .and_return() local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/root/.appscale/locations-bookey.json'),\ False,5)\ .and_return() # same for the secret key local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.secret'),False,5)\ .and_return() # mock out calls to the UserAppServer and presume that calls to create new # users succeed fake_userappserver = flexmock(name='fake_appcontroller') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( '[email protected]', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('set_cloud_admin_status').with_args( '*****@*****.**', 'true', 'the secret').and_return() fake_userappserver.should_receive('set_capabilities').with_args( '*****@*****.**', UserAppClient.ADMIN_CAPABILITIES, 'the secret').and_return() SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:4343') \ .and_return(fake_userappserver) # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_terminate_in_virtual_cluster_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # a virtualized cluster builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return( yaml.dump({'infrastructure': 'xen'})) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return( json.dumps([{ 'public_ip': 'public1', 'jobs': ['shadow'] }, { 'public_ip': 'public2', 'jobs': ['appengine'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # mock out talking to the appcontroller, and assume that it tells us there # there are still two machines in this deployment fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # and mock out the ssh call to kill the remote appcontroller, assuming that # it fails the first time and passes the second flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('controller stop'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.failed).and_return(self.success) # next, mock out our checks to see how the stopping process is going and # assume that it has stopped flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('ps x'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() # also mock out asking the user for confirmation on shutting down # their cloud builtins.should_receive('raw_input').and_return('yes') argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_terminate_in_gce_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return(True) os.path.should_receive('exists').with_args( LocalState.get_client_secrets_location(self.keyname)).and_return(True) os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # GCE project_id = "1234567890" zone = 'my-zone-1b' builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return(yaml.dump({ 'infrastructure' : 'gce', 'group' : self.group, 'project' : project_id, 'zone' : zone })) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return(json.dumps([ { 'public_ip' : 'public1', 'jobs' : ['shadow'] }, { 'public_ip' : 'public2', 'jobs' : ['appengine'] } ])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # also add in a fake client-secrets file for GCE client_secrets = LocalState.get_client_secrets_location(self.keyname) # mock out talking to GCE # first, mock out the oauth library calls fake_flow = flexmock(name='fake_flow') flexmock(oauth2client.client) oauth2client.client.should_receive('flow_from_clientsecrets').with_args( client_secrets, scope=str).and_return(fake_flow) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(None) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args(str).and_return( fake_storage) fake_credentials = flexmock(name='fake_credentials') flexmock(oauth2client.tools) oauth2client.tools.should_receive('run').with_args(fake_flow, fake_storage).and_return(fake_credentials) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) fake_gce = flexmock(name='fake_gce') # let's say that two instances are running instance_one_info = { u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': u'appscale-bazboogroup-one', u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'tags': {u'fingerprint': u'42WmSpB8rSM='}, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public1' }], u'networkIP': u'private1', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup', u'name': u'nic0' }] } instance_two_info = { u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': u'appscale-bazboogroup-two', u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'tags': {u'fingerprint': u'42WmSpB8rSM='}, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public1' }], u'networkIP': u'private1', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup', u'name': u'nic0' }] } list_instance_info = { u'items': [instance_one_info, instance_two_info], u'kind': u'compute#instanceList', u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances' } fake_list_instance_request = flexmock(name='fake_list_instance_request') fake_list_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(list_instance_info) fake_instances = flexmock(name='fake_instances') fake_instances.should_receive('list').with_args(project=project_id, filter="name eq appscale-bazboogroup-.*", zone=zone) \ .and_return(fake_list_instance_request) fake_gce.should_receive('instances').and_return(fake_instances) # And assume that we can kill both of our instances fine delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de' delete_instance_info_one = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_instance, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'startTime': u'2013-05-27T10:44:51.849-07:00', u'insertTime': u'2013-05-27T10:44:51.806-07:00', u'targetId': u'12912855597472179535', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f', u'operationType': u'delete', u'progress': 0, u'id': u'11114355109942058217', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de', u'user': u'*****@*****.**' } delete_instance_info_two = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_instance, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'startTime': u'2013-05-27T10:44:51.849-07:00', u'insertTime': u'2013-05-27T10:44:51.806-07:00', u'targetId': u'12912855597472179535', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f', u'operationType': u'delete', u'progress': 0, u'id': u'11114355109942058217', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de', u'user': u'*****@*****.**' } fake_delete_instance_request_one = flexmock(name='fake_delete_instance_request_one') fake_delete_instance_request_one.should_receive('execute').with_args( http=fake_authorized_http).and_return(delete_instance_info_one) fake_instances.should_receive('delete').with_args(project=project_id, zone=zone, instance='appscale-bazboogroup-one').and_return( fake_delete_instance_request_one) fake_delete_instance_request_two = flexmock(name='fake_delete_instance_request_two') fake_delete_instance_request_two.should_receive('execute').with_args( http=fake_authorized_http).and_return(delete_instance_info_two) fake_instances.should_receive('delete').with_args(project=project_id, zone=zone, instance='appscale-bazboogroup-two').and_return( fake_delete_instance_request_two) # mock out our waiting for the instances to be deleted all_done = { u'status' : u'DONE' } fake_instance_checker = flexmock(name='fake_instance_checker') fake_instance_checker.should_receive('execute').and_return(all_done) fake_blocker = flexmock(name='fake_blocker') fake_blocker.should_receive('get').with_args(project=project_id, operation=delete_instance, zone=zone).and_return( fake_instance_checker) fake_gce.should_receive('zoneOperations').and_return(fake_blocker) # mock out the call to delete the firewall delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654' fake_delete_firewall_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_firewall, u'startTime': u'2013-05-27T11:01:35.482-07:00', u'insertTime': u'2013-05-27T11:01:35.390-07:00', u'targetId': u'11748720697396371259', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20', u'operationType': u'delete', u'progress': 0, u'id': u'15574488986772298961', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654', u'user': u'*****@*****.**' } fake_delete_firewall_request = flexmock(name='fake_delete_firewall_request') fake_delete_firewall_request.should_receive('execute').and_return(fake_delete_firewall_info) fake_firewalls = flexmock(name='fake_firewalls') fake_firewalls.should_receive('delete').with_args(project=project_id, firewall=self.group).and_return(fake_delete_firewall_request) fake_gce.should_receive('firewalls').and_return(fake_firewalls) # mock out the call to make sure the firewall was deleted fake_firewall_checker = flexmock(name='fake_firewall_checker') fake_firewall_checker.should_receive('execute').and_return(all_done) fake_blocker.should_receive('get').with_args(project=project_id, operation=delete_firewall).and_return(fake_firewall_checker) fake_gce.should_receive('globalOperations').and_return(fake_blocker) # and the call to delete the network delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca' fake_delete_network_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_network, u'startTime': u'2013-05-27T11:02:30.012-07:00', u'insertTime': u'2013-05-27T11:02:29.954-07:00', u'targetId': u'17688075350400527692', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20', u'operationType': u'delete', u'progress': 0, u'id': u'12623697331874594836', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca', u'user': u'*****@*****.**' } fake_delete_network_request = flexmock(name='fake_delete_network_request') fake_delete_network_request.should_receive('execute').and_return(fake_delete_network_info) fake_networks = flexmock(name='fake_networks') fake_networks.should_receive('delete').with_args(project=project_id, network=self.group).and_return(fake_delete_network_request) fake_gce.should_receive('networks').and_return(fake_networks) # mock out the call to make sure the network was deleted fake_network_checker = flexmock(name='fake_network_checker') fake_network_checker.should_receive('execute').and_return(all_done) fake_blocker.should_receive('get').with_args(project=project_id, operation=delete_network).and_return(fake_network_checker) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive('build').with_args('compute', GCEAgent.API_VERSION).and_return(fake_gce) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() argv = [ "--keyname", self.keyname, "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running local_state = flexmock(LocalState) local_state.should_receive('ensure_appscale_isnt_running').and_return() local_state.should_receive('make_appscale_directory').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out interactions with AWS fake_ec2 = flexmock(name='fake_ec2') # first, pretend that our image does exist in EC2 fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \ .and_return() # next, assume that our keypair doesn't exist yet fake_ec2.should_receive('get_key_pair').with_args(self.keyname) \ .and_return(None) # same for the security group fake_ec2.should_receive('get_all_security_groups').and_return([]) # mock out creating the keypair fake_key = flexmock(name='fake_key', material='baz') local_state.should_receive('write_key_file').with_args( re.compile(self.keyname), fake_key.material).and_return() fake_ec2.should_receive('create_key_pair').with_args(self.keyname) \ .and_return(fake_key) # and the same for the security group fake_ec2.should_receive('create_security_group').with_args('bazgroup', str).and_return() fake_ec2.should_receive('authorize_security_group').with_args('bazgroup', from_port=1, to_port=65535, ip_protocol='udp', cidr_ip='0.0.0.0/0') fake_ec2.should_receive('authorize_security_group').with_args('bazgroup', from_port=1, to_port=65535, ip_protocol='tcp', cidr_ip='0.0.0.0/0') fake_ec2.should_receive('authorize_security_group').with_args('bazgroup', ip_protocol='icmp', cidr_ip='0.0.0.0/0') # also mock out acquiring a spot instance fake_ec2.should_receive('request_spot_instances').with_args('1.23', 'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'], instance_type='m1.large', count=1) # assume that there are no instances running initially, and that the # instance we spawn starts as pending, then becomes running no_instances = flexmock(name='no_instances', instances=[]) pending_instance = flexmock(name='pending_instance', state='pending', key_name=self.keyname, id='i-ABCDEFG') pending_reservation = flexmock(name='pending_reservation', instances=[pending_instance]) running_instance = flexmock(name='running_instance', state='running', key_name=self.keyname, id='i-ABCDEFG', public_dns_name='public1', private_dns_name='private1') running_reservation = flexmock(name='running_reservation', instances=[running_instance]) fake_ec2.should_receive('get_all_instances').and_return(no_instances) \ .and_return(pending_reservation).and_return(running_reservation) # finally, inject the mocked EC2 in flexmock(boto) boto.should_receive('connect_ec2').and_return(fake_ec2) # assume that root login is not enabled local_state.should_receive('shell').with_args(re.compile('ssh'), False, 1, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('sudo cp')).and_return() # and assume that we can copy over our ssh keys fine local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() # mock out seeing if the image is appscale-compatible, and assume it is # mock out our attempts to find /etc/appscale and presume it does exist local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('/etc/appscale')).and_return() # mock out our attempts to find /etc/appscale/version and presume it does # exist local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('/etc/appscale/{0}' .format(APPSCALE_VERSION))) # put in a mock indicating that the database the user wants is supported local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('/etc/appscale/{0}/{1}' .format(APPSCALE_VERSION, 'cassandra'))) # mock out generating the private key local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started god fine local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('god &')) # and that we copied over the AppController's god file local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('appcontroller.god')) # also, that we started the AppController itself local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('god load')) # assume that ssh comes up on the third attempt fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('public1', RemoteHelper.SSH_PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # assume that the AppController comes up on the third attempt fake_socket.should_receive('connect').with_args(('public1', AppControllerClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # same for the UserAppServer fake_socket.should_receive('connect').with_args(('public1', UserAppClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # as well as for the AppDashboard fake_socket.should_receive('connect').with_args(('public1', RemoteHelper.APP_DASHBOARD_PORT)).and_raise(Exception) \ .and_raise(Exception).and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_parameters').with_args(list, list, ['none'], 'the secret').and_return('OK') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1'])) role_info = [{ 'public_ip' : 'public1', 'private_ip' : 'private1', 'jobs' : ['shadow', 'login'] }] fake_appcontroller.should_receive('get_role_info').with_args('the secret') \ .and_return(json.dumps(role_info)) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at public1') fake_appcontroller.should_receive('is_done_initializing') \ .and_return(False) \ .and_return(True) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('set_cloud_admin_status').with_args( '*****@*****.**', 'true', 'the secret').and_return() fake_userappserver.should_receive('set_capabilities').with_args( '*****@*****.**', UserAppClient.ADMIN_CAPABILITIES, 'the secret').and_return() SOAPpy.should_receive('SOAPProxy').with_args('https://public1:4343') \ .and_return(fake_userappserver) argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", "bazgroup", "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_upload_tar_gz_app_successfully(self): # mock out generating a random app dir, for later mocks flexmock(uuid) uuid.should_receive('uuid4').and_return('12345678') app_dir = '/tmp/appscale-app-12345678' # add in mocks so that the gzip'ed file gets extracted to /tmp # as well as for removing it later flexmock(os) os.should_receive('mkdir').with_args(app_dir) \ .and_return(True) flexmock(shutil) shutil.should_receive('rmtree').with_args(app_dir).and_return() local_state = flexmock(LocalState) local_state.should_receive('shell')\ .with_args(re.compile('tar zxvf'),False)\ .and_return() # add in mocks so that there is an app.yaml, but with no appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return(yaml.dump({ 'application' : 'baz', 'runtime' : 'python' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('done_uploading').with_args('baz', '/var/apps/baz/app/baz.tar.gz', 'the secret').and_return() fake_appcontroller.should_receive('update').with_args(['baz'], 'the secret').and_return() fake_appcontroller.should_receive('is_app_running').with_args('baz', 'the secret').and_return(False).and_return(True) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return('\n\nnum_ports:0\n') \ .and_return(app_data).and_return(app_data).and_return(app_data) fake_userappserver.should_receive('commit_new_app').with_args( 'baz', '*****@*****.**', 'python', 'the secret').and_return('true') SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') # mock out making the remote app directory local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('^mkdir -p'))\ .and_return() # and mock out tarring and copying the app local_state.should_receive('shell')\ .with_args(re.compile('tar -czf'),False)\ .and_return() local_state.should_receive('shell')\ .with_args(re.compile('/tmp/appscale-app-baz.tar.gz'),False,5)\ .and_return() # as well as removing the tar'ed app once we're done copying it flexmock(os) os.should_receive('remove').with_args('/tmp/appscale-app-baz.tar.gz') \ .and_return() # and slap in a mock that says the app comes up after waiting for it # three times fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('public1', 8080)).and_raise(Exception).and_raise(Exception) \ .and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) argv = [ "--keyname", self.keyname, "--file", self.app_dir + ".tar.gz" ] options = ParseArgs(argv, self.function).args AppScaleTools.upload_app(options)
def test_upload_app_when_app_exists_on_virt_cluster(self): # we do let you upload an app if it's already running # add in mocks so that there is an app.yaml with an appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return(yaml.dump({ 'application' : 'baz', 'runtime' : 'python27' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('done_uploading').with_args( 'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return('OK') fake_appcontroller.should_receive('update').with_args( ['baz'], 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('does_user_exist').with_args( 'a@public1', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return(app_data) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') # mock out making the remote app directory flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # and mock out tarring and copying the app subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile( '/tmp/appscale-app-baz.tar.gz'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # as well as removing the tar'ed app once we're done copying it flexmock(os) os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \ .and_return() # and slap in a mock that says the app comes up after waiting for it # three times fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('public1', 8080)).and_raise(Exception).and_raise(Exception) \ .and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) argv = [ "--keyname", self.keyname, "--file", self.app_dir ] options = ParseArgs(argv, self.function).args (host, port) = AppScaleTools.upload_app(options) self.assertEquals('public1', host) self.assertEquals(8080, port)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running self.local_state.should_receive( 'ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args( 'POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args( '1.23', 'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'], instance_type='m3.medium', count=1, placement='my-zone-1b') # assume that root login is not enabled self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo touch /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='mktemp').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' )).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ') ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args( re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args( re.compile('scp .*{0}'.format(self.keyname)), False, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args( re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args( re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('public1') self.setup_appcontroller_mocks('public1', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return( json.loads( json.dumps([{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell').with_args( re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args( re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin= 'cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/' ).and_return() self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin= 'cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/' ).and_return() self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin= 'cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/' ) self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() self.setup_uaserver_mocks('public1') argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b" ] acc = flexmock(AppControllerClient) acc.should_receive('is_initialized').and_return(True) uac = flexmock(UserAppClient) uac.should_receive('does_user_exist').and_return(False) options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_terminate_in_virtual_cluster_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # a virtualized cluster builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return(yaml.dump({ 'infrastructure' : 'xen' })) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return(json.dumps([ { 'public_ip' : 'public1', 'jobs' : ['shadow'] }, { 'public_ip' : 'public2', 'jobs' : ['appengine'] } ])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # mock out talking to the appcontroller, and assume that it tells us there # there are still two machines in this deployment fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # and mock out the ssh call to kill the remote appcontroller, assuming that # it fails the first time and passes the second flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('controller stop'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.failed).and_return(self.success) # next, mock out our checks to see how the stopping process is going and # assume that it has stopped flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('ps x'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_upload_php_app_successfully(self): app_dir = '/tmp/appscale-app-1234' # add in mocks so that the gzip'ed file gets extracted to /tmp # as well as for removing it later flexmock(os) os.should_receive('mkdir').with_args(app_dir) \ .and_return(True) flexmock(shutil) shutil.should_receive('rmtree').with_args(app_dir).and_return() local_state = flexmock(LocalState) local_state.should_receive('shell')\ .with_args(re.compile('tar zxvf'),False)\ .and_return() # add in mocks so that there is an app.yaml, but with no appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return( yaml.dump({ 'application': 'baz', 'runtime': 'php' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('done_uploading').with_args( 'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return() fake_appcontroller.should_receive('update').with_args( ['baz'], 'the secret').and_return() fake_appcontroller.should_receive('is_app_running').with_args( 'baz', 'the secret').and_return(False).and_return(True) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('does_user_exist').with_args( 'a@public1', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return('\n\nnum_ports:0\n') \ .and_return(app_data).and_return(app_data).and_return(app_data) fake_userappserver.should_receive('commit_new_app').with_args( 'baz', '*****@*****.**', 'php', 'the secret').and_return('true') SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') # mock out making the remote app directory local_state.should_receive('shell') \ .with_args(re.compile('^ssh'), False, 5, stdin=re.compile('^mkdir -p')) \ .and_return() # and mock out tarring and copying the app local_state.should_receive('shell') \ .with_args(re.compile('tar -czf'), False) \ .and_return() local_state.should_receive('shell') \ .with_args(re.compile('/tmp/appscale-app-baz.tar.gz'), False, 5) \ .and_return() # as well as removing the tar'ed app once we're done copying it flexmock(os) os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \ .and_return() os.should_receive('listdir').and_return(['app.yaml', 'index.py']) # and slap in a mock that says the app comes up after waiting for it # three times fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('public1', 8080)).and_raise(Exception).and_raise(Exception) \ .and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) argv = ["--keyname", self.keyname, "--file", self.app_dir + ".tar.gz"] options = ParseArgs(argv, self.function).args (host, port) = AppScaleTools.upload_app(options) self.assertEquals('public1', host) self.assertEquals(8080, port)
def test_upload_app_when_app_admin_not_this_user(self): # we don't let you upload an app if the appid is registered to someone else, # so abort # add in mocks so that there is an app.yaml, but with no appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return( yaml.dump({ 'application': 'baz', 'runtime': 'python27' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('does_user_exist').with_args( 'a@public1', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return('\n\nnum_ports:0\n\napp_owner:[email protected]') SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') argv = ["--keyname", self.keyname, "--file", self.app_dir] options = ParseArgs(argv, self.function).args self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def test_upload_app_when_app_exists_on_virt_cluster(self): # we do let you upload an app if it's already running # add in mocks so that there is an app.yaml with an appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return( yaml.dump({ 'application': 'baz', 'runtime': 'python27' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('done_uploading').with_args( 'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return('OK') fake_appcontroller.should_receive('update').with_args( ['baz'], 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('does_user_exist').with_args( 'a@public1', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return(app_data) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') # mock out making the remote app directory flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # and mock out tarring and copying the app subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile( '/tmp/appscale-app-baz.tar.gz'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) # as well as removing the tar'ed app once we're done copying it flexmock(os) os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \ .and_return() # and slap in a mock that says the app comes up after waiting for it # three times fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('public1', 8080)).and_raise(Exception).and_raise(Exception) \ .and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) argv = ["--keyname", self.keyname, "--file", self.app_dir] options = ParseArgs(argv, self.function).args (host, port) = AppScaleTools.upload_app(options) self.assertEquals('public1', host) self.assertEquals(8080, port)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args('1.23', 'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'], instance_type='m1.large', count=1, placement='my-zone-1b') # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('sudo cp')).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('public1') self.setup_appcontroller_mocks('public1', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.setup_uaserver_mocks('public1') argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self): # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # slip in some fake spot instance info fake_entry = flexmock(name='fake_entry', price=1) self.fake_ec2.should_receive('get_spot_price_history').with_args( start_time=str, end_time=str, product_description='Linux/UNIX', instance_type='m1.large', availability_zone='my-zone-1b').and_return([fake_entry]) # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args('1.1', 'ami-ABCDEFG', key_name=self.keyname, security_groups=[self.group], instance_type='m1.large', count=1, placement='my-zone-1b') # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ') ).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('elastic-ip') self.setup_appcontroller_mocks('elastic-ip', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "elastic-ip", "private_ip" : "private1", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.setup_uaserver_mocks('elastic-ip') argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b", "--static_ip", "elastic-ip" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_gce_deployment(self): # presume that our client_secrets file exists project_id = "appscale.com:appscale" client_secrets = "/boo/client_secrets.json" instance_type = 'n1-standard-8' zone = 'my-zone1-b' os.path.should_receive('exists').with_args(client_secrets).and_return(True) # and that the user does not have an ssh key set up, forcing us to create # one for them private_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, self.keyname) public_key = '{0}{1}.pub'.format(LocalState.LOCAL_APPSCALE_PATH, self.keyname) os.path.should_receive('exists').with_args(private_key).and_return(False) os.path.should_receive('exists').with_args(public_key).and_return(False) self.local_state.should_receive('shell').with_args(re.compile('^ssh-keygen'), False).and_return() flexmock(os) original_private_key = LocalState.LOCAL_APPSCALE_PATH + self.keyname os.should_receive('chmod').with_args(original_private_key, 0600) os.should_receive('chmod').with_args(public_key, 0600) flexmock(shutil) shutil.should_receive('copy').with_args(original_private_key, private_key) # also, we should be able to copy over our secret.json file fine shutil.should_receive('copy').with_args(client_secrets, LocalState.get_client_secrets_location(self.keyname)) # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out interactions with GCE # first, mock out the oauth library calls fake_flow = flexmock(name='fake_flow') flexmock(oauth2client.client) oauth2client.client.should_receive('flow_from_clientsecrets').with_args( client_secrets, scope=str).and_return(fake_flow) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(None) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args(str).and_return( fake_storage) fake_credentials = flexmock(name='fake_credentials') flexmock(oauth2client.tools) oauth2client.tools.should_receive('run').with_args(fake_flow, fake_storage).and_return(fake_credentials) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) # presume that there is an ssh key stored, but it isn't ours metadata_info = { u'kind': u'compute#project', u'description': u'', u'commonInstanceMetadata': { u'items': [{ u'value': u'cgb:ssh-rsa keyinfo myhost', u'key': u'sshKeys'}], u'kind': u'compute#metadata'}, } fake_metadata_request = flexmock(name='fake_metadata_request') fake_metadata_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(metadata_info) fake_projects = flexmock(name='fake_projects') fake_projects.should_receive('get').with_args(project=project_id) \ .and_return(fake_metadata_request) fake_gce = flexmock(name='fake_gce') fake_gce.should_receive('projects').and_return(fake_projects) # thus we will need to set the metadata with our ssh key fake_ssh_pub_key = flexmock(name="fake_ssh_pub_key") fake_ssh_pub_key.should_receive('read').and_return('ssh-rsa key2info myhost') self.builtins.should_receive('open').with_args(public_key).and_return( fake_ssh_pub_key) new_metadata_body = { "items": [{ "value" : u'cgb:ssh-rsa key2info myhost\ncgb:ssh-rsa keyinfo myhost', "key" : "sshKeys" }], "kind": "compute#metadata" } set_metadata_name = u'operation-222222-4dd41ec7d6c11-8013657f' set_metadata = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': set_metadata_name, u'operationType': u'insert', u'progress': 0, u'selfLink': unicode(GCEAgent.GCE_URL) + \ u'appscale.com:appscale/global/operations/' + \ u'operation-1369175117235-4dd41ec7d6c11-8013657f', u'user': u'*****@*****.**' } fake_set_metadata_request = flexmock(name='fake_set_metadata_request') fake_set_metadata_request.should_receive('execute').and_return(set_metadata) fake_projects.should_receive('setCommonInstanceMetadata').with_args( project=project_id, body=dict).and_return(fake_set_metadata_request) updated_metadata_info = { u'status': u'DONE' } fake_metadata_checker = flexmock(name='fake_network_checker') fake_metadata_checker.should_receive('execute').and_return( updated_metadata_info) fake_blocker = flexmock(name='fake_blocker') fake_blocker.should_receive('get').with_args(project=project_id, operation=set_metadata_name).and_return(fake_metadata_checker) # presume that our image does exist in GCE, with some fake data # acquired by running a not mocked version of this code image_name = 'appscale-image-name' image_info = { u'kind': u'compute#image', u'description': u'', u'rawDisk': {u'containerType': u'TAR', u'source': u''}, u'preferredKernel': unicode(GCEAgent.GCE_URL) + \ u'/google/global/kernels/gce-v20130515', u'sourceType': u'RAW', u'creationTimestamp': u'2013-05-21T08:05:12.198-07:00', u'id': u'4235320207849085220', u'selfLink': unicode(GCEAgent.GCE_URL) + \ u'961228229472/global/images/' + unicode(image_name), u'name': unicode(image_name) } fake_image_request = flexmock(name='fake_image_request') fake_image_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(image_info) fake_images = flexmock(name='fake_images') fake_images.should_receive('get').with_args(project=project_id, image=image_name).and_return(fake_image_request) fake_gce.should_receive('images').and_return(fake_images) # next, presume that the static ip we want to use exists address_name = 'static-ip' address_info = {'items':[]} region_name = 'my-zone1' fake_address_request = flexmock(name='fake_address_request') fake_address_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(address_info) fake_addresses = flexmock(name='fake_addresses') fake_addresses.should_receive('list').with_args(project=project_id, filter="address eq static-ip", region=region_name).and_return( fake_address_request) fake_gce.should_receive('addresses').and_return(fake_addresses) # next, presume that the zone we want to use exists zone_name = 'my-zone1-b' zone_info = {} fake_zone_request = flexmock(name='fake_zone_request') fake_zone_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(zone_info) fake_zones = flexmock(name='fake_zones') fake_zones.should_receive('get').with_args(project=project_id, zone=zone_name).and_return(fake_zone_request) fake_gce.should_receive('zones').and_return(fake_zones) # next, presume that the persistent disk we want to use exists disk_name = 'my-persistent-disk-1' disk_info = {'status':'DONE'} fake_disk_request = flexmock(name='fake_disk_request') fake_disk_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(disk_info) fake_disks = flexmock(name='fake_disks') fake_disks.should_receive('get').with_args(project=project_id, disk=disk_name, zone=zone).and_return(fake_disk_request) fake_disks.should_receive('insert').with_args(project=project_id, sourceImage=str, body=dict, zone=zone_name).and_return(fake_disk_request) fake_gce.should_receive('disks').and_return(fake_disks) # next, presume that the network doesn't exist yet fake_network_request = flexmock(name='fake_network_request') fake_network_request.should_receive('execute').with_args( http=fake_authorized_http).and_raise(apiclient.errors.HttpError, None, None) fake_networks = flexmock(name='fake_networks') fake_networks.should_receive('get').with_args(project=project_id, network='bazgroup').and_return(fake_network_request) fake_gce.should_receive('networks').and_return(fake_networks) # next, presume that the firewall doesn't exist yet fake_firewall_request = flexmock(name='fake_firewall_request') fake_firewall_request.should_receive('execute').with_args( http=fake_authorized_http).and_raise(apiclient.errors.HttpError, None, None) fake_firewalls = flexmock(name='fake_firewalls') fake_firewalls.should_receive('get').with_args(project=project_id, firewall='bazgroup').and_return(fake_firewall_request) fake_gce.should_receive('firewalls').and_return(fake_firewalls) # presume that we can create the network fine create_network = u'operation-1369175117235-4dd41ec7d6c11-8013657f' network_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': create_network, u'startTime': u'2013-05-21T15:25:17.308-07:00', u'insertTime': u'2013-05-21T15:25:17.235-07:00', u'targetLink': unicode(GCEAgent.GCE_URL) + \ u'appscale.com:appscale/global/networks/bazgroup', u'operationType': u'insert', u'progress': 0, u'id': u'4904874319704759670', u'selfLink': unicode(GCEAgent.GCE_URL) + \ u'appscale.com:appscale/global/operations/' + \ u'operation-1369175117235-4dd41ec7d6c11-8013657f', u'user': u'*****@*****.**' } fake_network_insert_request = flexmock(name='fake_network_insert_request') fake_network_insert_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(network_info) fake_networks.should_receive('insert').with_args(project=project_id, body=dict).and_return(fake_network_insert_request) created_network_info = { u'status': u'DONE' } fake_network_checker = flexmock(name='fake_network_checker') fake_network_checker.should_receive('execute').and_return( created_network_info) fake_blocker.should_receive('get').with_args(project=project_id, operation=create_network).and_return(fake_network_checker) fake_gce.should_receive('globalOperations').and_return(fake_blocker) # and presume that we can create the firewall fine create_firewall = u'operation-1369176378310-4dd4237a84021-68e4dfa6' firewall_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': create_firewall, u'startTime': u'2013-05-21T15:46:18.402-07:00', u'insertTime': u'2013-05-21T15:46:18.310-07:00', u'targetLink': unicode(GCEAgent.GCE_URL) + \ u'appscale.com:appscale/global/firewalls/bazgroup', u'operationType': u'insert', u'progress': 0, u'id': u'13248349431060541723', u'selfLink': unicode(GCEAgent.GCE_URL) + \ u'appscale.com:appscale/global/operations/' + \ u'operation-1369176378310-4dd4237a84021-68e4dfa6', u'user': u'*****@*****.**' } fake_firewall_insert_request = flexmock(name='fake_firewall_insert_request') fake_firewall_insert_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(firewall_info) fake_firewalls.should_receive('insert').with_args(project= u'appscale.com:appscale', body=dict).and_return(fake_firewall_insert_request) created_firewall_info = { u'status': u'DONE' } fake_firewall_checker = flexmock(name='fake_network_checker') fake_firewall_checker.should_receive('execute').and_return( created_firewall_info) fake_blocker.should_receive('get').with_args(project=project_id, operation=create_firewall).and_return(fake_firewall_checker) # we only need to create one node, so set up mocks for that add_instance = u'operation-1369248752891-4dd5311848461-afc55a20' instance_id = 'appscale-bazgroup-feb10b11-62bc-4536-ac25-9734f2267d6d' add_instance_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': add_instance, u'azone': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a', u'startTime': u'2013-05-22T11:52:32.939-07:00', u'insertTime': u'2013-05-22T11:52:32.891-07:00', u'targetLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/instances/' + instance_id, u'operationType': u'insert', u'progress': 0, u'id': u'6663616273628949255', u'selfLink': unicode(GCEAgent.GCE_URL) + u'appscale.com:appscale/zones/us-central1-a/operations/operation-1369248752891-4dd5311848461-afc55a20', u'user': u'*****@*****.**' } fake_add_instance_request = flexmock(name='fake_add_instance_request') fake_add_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(add_instance_info) fake_instances = flexmock(name='fake_instances') fake_gce.should_receive('instances').and_return(fake_instances) fake_instances.should_receive('insert').with_args(project=project_id, body=dict, zone=str).and_return(fake_add_instance_request) created_instance_info = { u'status': u'DONE' } fake_instance_checker = flexmock(name='fake_network_checker') fake_instance_checker.should_receive('execute').and_return( created_instance_info) fake_blocker.should_receive('get').with_args(project=project_id, operation=add_instance).and_return(fake_instance_checker) # add some fake data in where no instances are initially running, then one # is (in response to our insert request) no_instance_info = { } list_instance_info = { u'items': [{ u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a/machineTypes/' + instance_type, u'name': instance_id, u'zone': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a', u'tags': {u'fingerprint': u'42WmSpB8rSM='}, u'image': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/zones/us-central1-a/instances/' + instance_id, u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public1' }], u'networkIP': u'private1', u'network': u'https://www.googleapis.com/compute/v1/projects/appscale.com:appscale/global/networks/bazgroup', u'name': u'nic0' }] }], u'kind': u'compute#instanceList', u'id': u'projects/appscale.com:appscale/zones/us-central1-a/instances', u'selfLink': u'https://www.googleapis.com/compute/v1/projects/961228229472/zones/us-central1-a/instances' } fake_list_instance_request = flexmock(name='fake_list_instance_request') fake_list_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(no_instance_info).and_return( no_instance_info).and_return(list_instance_info) fake_instances.should_receive('list').with_args(project=project_id, filter="name eq appscale-bazgroup-.*", zone=zone) \ .and_return(fake_list_instance_request) fake_instances.should_receive('list').with_args(project=project_id, zone=zone).and_return(fake_list_instance_request) # mock out deleting the public IP from the instance fake_delete_access_request = flexmock(name='fake_delete_access_request') fake_delete_access_request.should_receive('execute').with_args( http=fake_authorized_http).and_return() fake_instances.should_receive('deleteAccessConfig').with_args( project=project_id, accessConfig=str, instance=instance_id, networkInterface=str, zone=zone).and_return(fake_delete_access_request) # as well as adding in the new, static IP to the instance fake_add_access_request = flexmock(name='fake_add_access_request') fake_add_access_request.should_receive('execute').with_args( http=fake_authorized_http).and_return() fake_instances.should_receive('addAccessConfig').with_args( project=project_id, instance=instance_id, networkInterface=str, zone=zone, body=dict).and_return(fake_add_access_request) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive('build').with_args('compute', str) \ .and_return(fake_gce) # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_raise(ShellException) # assume that we can enable root login self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('sudo cp')).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('static-ip') self.setup_appcontroller_mocks('static-ip', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "static-ip", "private_ip" : "private1", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.setup_uaserver_mocks('static-ip') # Finally, pretend we're using a single persistent disk. disk_layout = yaml.safe_load(""" node-1: my-persistent-disk-1 """) argv = [ "--min", "1", "--max", "1", "--disks", base64.b64encode(yaml.dump(disk_layout)), "--group", self.group, "--infrastructure", "gce", "--gce_instance_type", instance_type, "--machine", image_name, "--keyname", self.keyname, "--client_secrets", client_secrets, "--project", project_id, "--test", "--zone", "my-zone1-b", "--static_ip", "static-ip" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_terminate_in_cloud_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # a virtualized cluster builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return( yaml.dump({ 'infrastructure': 'ec2', 'group': self.group, })) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return( json.dumps([{ 'public_ip': 'public1', 'jobs': ['shadow'] }, { 'public_ip': 'public2', 'jobs': ['appengine'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # mock out talking to EC2 fake_ec2 = flexmock(name='fake_ec2') # let's say that three instances are running, and that two of them are in # our deployment fake_one_running = flexmock(name='fake_one', key_name=self.keyname, state='running', id='i-ONE', public_dns_name='public1', private_dns_name='private1') fake_two_running = flexmock(name='fake_two', key_name=self.keyname, state='running', id='i-TWO', public_dns_name='public2', private_dns_name='private2') fake_three_running = flexmock(name='fake_three', key_name='abcdefg', state='running', id='i-THREE', public_dns_name='public3', private_dns_name='private3') fake_reservation_running = flexmock( name='fake_reservation', instances=[fake_one_running, fake_two_running, fake_three_running]) fake_one_terminated = flexmock(name='fake_one', key_name=self.keyname, state='terminated', id='i-ONE', public_dns_name='public1', private_dns_name='private1') fake_two_terminated = flexmock(name='fake_two', key_name=self.keyname, state='terminated', id='i-TWO', public_dns_name='public2', private_dns_name='private2') fake_three_terminated = flexmock(name='fake_three', key_name='abcdefg', state='terminated', id='i-THREE', public_dns_name='public3', private_dns_name='private3') fake_reservation_terminated = flexmock(name='fake_reservation', instances=[ fake_one_terminated, fake_two_terminated, fake_three_terminated ]) fake_ec2.should_receive('get_all_instances').and_return(fake_reservation_running) \ .and_return(fake_reservation_terminated) flexmock(boto.ec2) boto.ec2.should_receive('connect_to_region').and_return(fake_ec2) # and mock out the call to kill the instances fake_ec2.should_receive('terminate_instances').with_args( ['i-ONE', 'i-TWO']).and_return([fake_one_terminated, fake_two_terminated]) # mock out the call to delete the keypair fake_ec2.should_receive('delete_key_pair').and_return() # and the call to delete the security group - let's say that we can't # delete the group the first time, and can the second fake_ec2.should_receive('delete_security_group').and_return(False) \ .and_return(True) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() # also mock out asking the user for confirmation on shutting down # their cloud builtins.should_receive('raw_input').and_return('yes') argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_upload_app_when_app_admin_not_this_user(self): # we don't let you upload an app if the appid is registered to someone else, # so abort # add in mocks so that there is an app.yaml, but with no appid set flexmock(os.path) os.path.should_call('exists') app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir) os.path.should_receive('exists').with_args(app_yaml_location) \ .and_return(True) # mock out reading the app.yaml file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through fake_app_yaml = flexmock(name="fake_app_yaml") fake_app_yaml.should_receive('read').and_return(yaml.dump({ 'application' : 'baz', 'runtime' : 'python' })) builtins.should_receive('open').with_args(app_yaml_location, 'r') \ .and_return(fake_app_yaml) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( 'a@public1', str, 'xmpp_user', 'the secret').and_return('true') fake_userappserver.should_receive('get_app_data').with_args( 'baz', 'the secret').and_return('\n\nnum_ports:0\n\napp_owner:[email protected]') SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com") flexmock(getpass) getpass.should_receive('getpass').and_return('aaaaaa') argv = [ "--keyname", self.keyname, "--file", self.app_dir ] options = ParseArgs(argv, self.function).args self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def test_terminate_in_gce_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_yaml_location( self.keyname)).and_return(True) os.path.should_receive('exists').with_args( LocalState.get_client_secrets_location( self.keyname)).and_return(True) os.path.should_receive('exists').with_args( LocalState.get_secret_key_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # GCE project_id = "1234567890" zone = 'my-zone-1b' builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return( yaml.dump({ 'infrastructure': 'gce', 'group': self.group, 'project': project_id, 'zone': zone })) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return( json.dumps([{ 'public_ip': 'public1', 'jobs': ['shadow'] }, { 'public_ip': 'public2', 'jobs': ['appengine'] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # also add in a fake client-secrets file for GCE client_secrets = LocalState.get_client_secrets_location(self.keyname) # mock out talking to GCE # first, mock out the oauth library calls fake_flow = flexmock(name='fake_flow') flexmock(oauth2client.client) oauth2client.client.should_receive( 'flow_from_clientsecrets').with_args( client_secrets, scope=str).and_return(fake_flow) fake_storage = flexmock(name='fake_storage') fake_storage.should_receive('get').and_return(None) fake_flags = oauth2client.tools.argparser.parse_args(args=[]) flexmock(oauth2client.file) oauth2client.file.should_receive('Storage').with_args(str).and_return( fake_storage) fake_credentials = flexmock(name='fake_credentials') flexmock(oauth2client.tools) oauth2client.tools.should_receive('run_flow').with_args( fake_flow, fake_storage, fake_flags).and_return(fake_credentials) # next, mock out http calls to GCE fake_http = flexmock(name='fake_http') fake_authorized_http = flexmock(name='fake_authorized_http') flexmock(httplib2) httplib2.should_receive('Http').and_return(fake_http) fake_credentials.should_receive('authorize').with_args(fake_http) \ .and_return(fake_authorized_http) fake_gce = flexmock(name='fake_gce') # let's say that two instances are running instance_one_info = { u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': u'appscale-bazboogroup-one', u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'tags': { u'fingerprint': u'42WmSpB8rSM=' }, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public1' }], u'networkIP': u'private1', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup', u'name': u'nic0' }] } instance_two_info = { u'status': u'RUNNING', u'kind': u'compute#instance', u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1', u'name': u'appscale-bazboogroup-two', u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'tags': { u'fingerprint': u'42WmSpB8rSM=' }, u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64', u'disks': [{ u'index': 0, u'kind': u'compute#attachedDisk', u'type': u'EPHEMERAL', u'mode': u'READ_WRITE' }], u'canIpForward': False, u'serviceAccounts': [{ u'scopes': [GCEAgent.GCE_SCOPE], u'email': u'*****@*****.**' }], u'metadata': { u'kind': u'compute#metadata', u'fingerprint': u'42WmSpB8rSM=' }, u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00', u'id': u'8684033495853907982', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d', u'networkInterfaces': [{ u'accessConfigs': [{ u'kind': u'compute#accessConfig', u'type': u'ONE_TO_ONE_NAT', u'name': u'External NAT', u'natIP': u'public1' }], u'networkIP': u'private1', u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup', u'name': u'nic0' }] } list_instance_info = { u'items': [instance_one_info, instance_two_info], u'kind': u'compute#instanceList', u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances' } fake_list_instance_request = flexmock( name='fake_list_instance_request') fake_list_instance_request.should_receive('execute').with_args( http=fake_authorized_http).and_return(list_instance_info) fake_instances = flexmock(name='fake_instances') fake_instances.should_receive('list').with_args(project=project_id, filter="name eq appscale-bazboogroup-.*", zone=zone) \ .and_return(fake_list_instance_request) fake_gce.should_receive('instances').and_return(fake_instances) # And assume that we can kill both of our instances fine delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de' delete_instance_info_one = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_instance, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'startTime': u'2013-05-27T10:44:51.849-07:00', u'insertTime': u'2013-05-27T10:44:51.806-07:00', u'targetId': u'12912855597472179535', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f', u'operationType': u'delete', u'progress': 0, u'id': u'11114355109942058217', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de', u'user': u'*****@*****.**' } delete_instance_info_two = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_instance, u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b', u'startTime': u'2013-05-27T10:44:51.849-07:00', u'insertTime': u'2013-05-27T10:44:51.806-07:00', u'targetId': u'12912855597472179535', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscale-appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f', u'operationType': u'delete', u'progress': 0, u'id': u'11114355109942058217', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de', u'user': u'*****@*****.**' } fake_delete_instance_request_one = flexmock( name='fake_delete_instance_request_one') fake_delete_instance_request_one.should_receive('execute').with_args( http=fake_authorized_http).and_return(delete_instance_info_one) fake_instances.should_receive('delete').with_args( project=project_id, zone=zone, instance='appscale-bazboogroup-one').and_return( fake_delete_instance_request_one) fake_delete_instance_request_two = flexmock( name='fake_delete_instance_request_two') fake_delete_instance_request_two.should_receive('execute').with_args( http=fake_authorized_http).and_return(delete_instance_info_two) fake_instances.should_receive('delete').with_args( project=project_id, zone=zone, instance='appscale-bazboogroup-two').and_return( fake_delete_instance_request_two) # mock out our waiting for the instances to be deleted all_done = {u'status': u'DONE'} fake_instance_checker = flexmock(name='fake_instance_checker') fake_instance_checker.should_receive('execute').and_return(all_done) fake_blocker = flexmock(name='fake_blocker') fake_blocker.should_receive('get').with_args( project=project_id, operation=delete_instance, zone=zone).and_return(fake_instance_checker) fake_gce.should_receive('zoneOperations').and_return(fake_blocker) # mock out the call to delete the firewall delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654' fake_delete_firewall_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_firewall, u'startTime': u'2013-05-27T11:01:35.482-07:00', u'insertTime': u'2013-05-27T11:01:35.390-07:00', u'targetId': u'11748720697396371259', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20', u'operationType': u'delete', u'progress': 0, u'id': u'15574488986772298961', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654', u'user': u'*****@*****.**' } fake_delete_firewall_request = flexmock( name='fake_delete_firewall_request') fake_delete_firewall_request.should_receive('execute').and_return( fake_delete_firewall_info) fake_firewalls = flexmock(name='fake_firewalls') fake_firewalls.should_receive('delete').with_args( project=project_id, firewall=self.group).and_return(fake_delete_firewall_request) fake_gce.should_receive('firewalls').and_return(fake_firewalls) # mock out the call to make sure the firewall was deleted fake_firewall_checker = flexmock(name='fake_firewall_checker') fake_firewall_checker.should_receive('execute').and_return(all_done) fake_blocker.should_receive('get').with_args( project=project_id, operation=delete_firewall).and_return(fake_firewall_checker) fake_gce.should_receive('globalOperations').and_return(fake_blocker) # and the call to delete the network delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca' fake_delete_network_info = { u'status': u'PENDING', u'kind': u'compute#operation', u'name': delete_network, u'startTime': u'2013-05-27T11:02:30.012-07:00', u'insertTime': u'2013-05-27T11:02:29.954-07:00', u'targetId': u'17688075350400527692', u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20', u'operationType': u'delete', u'progress': 0, u'id': u'12623697331874594836', u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca', u'user': u'*****@*****.**' } fake_delete_network_request = flexmock( name='fake_delete_network_request') fake_delete_network_request.should_receive('execute').and_return( fake_delete_network_info) fake_networks = flexmock(name='fake_networks') fake_networks.should_receive('delete').with_args( project=project_id, network=self.group).and_return(fake_delete_network_request) fake_gce.should_receive('networks').and_return(fake_networks) # mock out the call to make sure the network was deleted fake_network_checker = flexmock(name='fake_network_checker') fake_network_checker.should_receive('execute').and_return(all_done) fake_blocker.should_receive('get').with_args( project=project_id, operation=delete_network).and_return(fake_network_checker) # finally, inject our fake GCE connection flexmock(apiclient.discovery) apiclient.discovery.should_receive('build').with_args( 'compute', GCEAgent.API_VERSION).and_return(fake_gce) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() argv = ["--keyname", self.keyname, "--test"] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_appscale_in_one_node_virt_deployment(self): self.local_state.should_receive('shell').with_args( "ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin= "cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/" ) self.local_state.should_receive('shell').with_args( "ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] ", False, 5, stdin="chmod +x /etc/init.d/appcontroller") self.local_state.should_receive('shell').with_args( "ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin= "cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/" ) # let's say that appscale isn't already running self.local_state.should_receive( 'ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() rh = flexmock(RemoteHelper) rh.should_receive('copy_deployment_credentials').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args( 'POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out copying over the keys self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.key'),False,5) self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell')\ .with_args(re.compile('^openssl'),False,stdin=None)\ .and_return() # mock out removing the old json file self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\ .and_return() # assume that we started monit fine self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\ .and_return() # and that we copied over the AppController's monit file self.local_state.should_receive('shell')\ .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\ .and_return() self.local_state.should_receive('shell').with_args( 'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null [email protected] ', False, 5, stdin= 'cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/' ).and_return() self.setup_socket_mocks('1.2.3.4') self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return( json.loads( json.dumps([{ "public_ip": "1.2.3.4", "private_ip": "1.2.3.4", "jobs": ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.yaml'),\ False,5)\ .and_return() self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.json'),\ False,5)\ .and_return() self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/root/.appscale/locations-bookey.json'),\ False,5)\ .and_return() # same for the secret key self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.secret'),False,5)\ .and_return() self.setup_uaserver_mocks('1.2.3.4') # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test" ] acc = flexmock(AppControllerClient) acc.should_receive('is_initialized').and_return(True) uac = flexmock(UserAppClient) uac.should_receive('does_user_exist').and_return(False) options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_add_nodes_in_virt_deployment(self): # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_yaml = """ database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """ # mock out reading the yaml file, and slip in our own fake_yaml_file = flexmock(name='fake_yaml') fake_yaml_file.should_receive('read').and_return(ips_yaml) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through builtins.should_receive('open').with_args('/tmp/boo.yaml', 'r') \ .and_return(fake_yaml_file) # and pretend we're on a virtualized cluster locations_yaml = yaml.dump({ "infrastructure" : "xen" }) fake_locations_yaml_file = flexmock(name='fake_yaml') fake_locations_yaml_file.should_receive('read').and_return(locations_yaml) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_locations_yaml_file) # say that the ssh key works flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('ssh'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT, stdin=self.fake_input_file) \ .and_return(self.success) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded json_node_info = json.dumps(yaml.safe_load(ips_yaml)) fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('start_roles_on_nodes') \ .with_args(json_node_info, 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = [ "--ips", "/tmp/boo.yaml", "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.add_instances(options)
def test_terminate_in_cloud_and_succeeds(self): # let's say that there is a locations.yaml file, which means appscale is # running, so we should terminate the services on each box flexmock(os.path) os.path.should_call('exists') # set up the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return(True) # mock out reading the locations.yaml file, and pretend that we're on # a virtualized cluster builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') fake_yaml_file = flexmock(name='fake_file') fake_yaml_file.should_receive('read').and_return(yaml.dump({ 'infrastructure' : 'ec2', 'group' : 'bazboogroup' })) builtins.should_receive('open').with_args( LocalState.get_locations_yaml_location(self.keyname), 'r') \ .and_return(fake_yaml_file) # mock out reading the json file, and pretend that we're running in a # two node deployment fake_json_file = flexmock(name='fake_file') fake_json_file.should_receive('read').and_return(json.dumps([ { 'public_ip' : 'public1', 'jobs' : ['shadow'] }, { 'public_ip' : 'public2', 'jobs' : ['appengine'] } ])) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_json_file) # and slip in a fake secret file fake_secret_file = flexmock(name='fake_file') fake_secret_file.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args( LocalState.get_secret_key_location(self.keyname), 'r') \ .and_return(fake_secret_file) # mock out talking to EC2 fake_ec2 = flexmock(name='fake_ec2') # let's say that three instances are running, and that two of them are in # our deployment fake_one = flexmock(name='fake_one', key_name=self.keyname, state='running', id='i-ONE', public_dns_name='public1', private_dns_name='private1') fake_two = flexmock(name='fake_two', key_name=self.keyname, state='running', id='i-TWO', public_dns_name='public2', private_dns_name='private2') fake_three = flexmock(name='fake_three', key_name='abcdefg', state='running', id='i-THREE', public_dns_name='public3', private_dns_name='private3') fake_reservation = flexmock(name='fake_reservation', instances=[fake_one, fake_two, fake_three]) fake_ec2.should_receive('get_all_instances').and_return(fake_reservation) flexmock(boto) boto.should_receive('connect_ec2').with_args('baz', 'baz') \ .and_return(fake_ec2) # and mock out the call to kill the instances fake_ec2.should_receive('terminate_instances').with_args(['i-ONE', 'i-TWO']).and_return([fake_one, fake_two]) # mock out the call to delete the keypair fake_ec2.should_receive('delete_key_pair').and_return() # and the call to delete the security group - let's say that we can't # delete the group the first time, and can the second fake_ec2.should_receive('delete_security_group').and_return(False) \ .and_return(True) # finally, mock out removing the yaml file, json file, and secret key from # this machine flexmock(os) os.should_receive('remove').with_args( LocalState.get_locations_yaml_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_locations_json_location(self.keyname)).and_return() os.should_receive('remove').with_args( LocalState.get_secret_key_location(self.keyname)).and_return() argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_appscale_in_one_node_virt_deployment(self): # let's say that appscale isn't already running local_state = flexmock(LocalState) local_state.should_receive('ensure_appscale_isnt_running').and_return() local_state.should_receive('make_appscale_directory').and_return() rh = flexmock(RemoteHelper) rh.should_receive('copy_deployment_credentials').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) # mock out copying over the keys local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.key'),False,5) # mock out our attempts to find /etc/appscale and presume it does exist local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale'))\ .and_return() # mock out our attempts to find /etc/appscale/version and presume it does # exist local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale/{0}'.format(APPSCALE_VERSION)))\ .and_return() # finally, put in a mock indicating that the database the user wants # is supported local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('ls /etc/appscale/{0}/{1}'\ .format(APPSCALE_VERSION, 'cassandra')))\ .and_return() # mock out generating the private key local_state.should_receive('shell')\ .with_args(re.compile('^openssl'),False,stdin=None)\ .and_return() # mock out removing the old json file local_state = flexmock(LocalState) local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\ .and_return() # assume that we started god fine local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('god &'))\ .and_return() # and that we copied over the AppController's god file local_state.should_receive('shell')\ .with_args(re.compile('scp .*appcontroller\.god.*'),False,5)\ .and_return() # also, that we started the AppController itself local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,\ stdin=re.compile('^god load .*appcontroller\.god'))\ .and_return() # assume that the AppController comes up on the third attempt fake_socket = flexmock(name='fake_socket') fake_socket.should_receive('connect').with_args(('1.2.3.4', AppControllerClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # same for the UserAppServer fake_socket.should_receive('connect').with_args(('1.2.3.4', UserAppClient.PORT)).and_raise(Exception).and_raise(Exception) \ .and_return(None) # as well as for the AppLoadBalancer fake_socket.should_receive('connect').with_args(('1.2.3.4', RemoteHelper.APP_LOAD_BALANCER_PORT)).and_raise(Exception) \ .and_raise(Exception).and_return(None) flexmock(socket) socket.should_receive('socket').and_return(fake_socket) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_parameters').with_args(list, list, ['none'], 'the secret').and_return('OK') fake_appcontroller.should_receive('get_all_public_ips')\ .with_args('the secret') \ .and_return(json.dumps(['1.2.3.4'])) role_info = [{ 'public_ip' : '1.2.3.4', 'private_ip' : '1.2.3.4', 'jobs' : ['shadow', 'login'] }] fake_appcontroller.should_receive('get_role_info').with_args('the secret') \ .and_return(json.dumps(role_info)) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') fake_appcontroller.should_receive('is_done_initializing') \ .and_return(False) \ .and_return(True) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "1.2.3.4", "private_ip" : "1.2.3.4", "jobs" : ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.yaml'),\ False,5)\ .and_return() local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/etc/appscale/locations-bookey.json'),\ False,5)\ .and_return() local_state.should_receive('shell')\ .with_args(re.compile('^scp .*/root/.appscale/locations-bookey.json'),\ False,5)\ .and_return() # same for the secret key local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.secret'),False,5)\ .and_return() # mock out calls to the UserAppServer and presume that calls to create new # users succeed fake_userappserver = flexmock(name='fake_appcontroller') fake_userappserver.should_receive('commit_new_user').with_args( '*****@*****.**', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('commit_new_user').with_args( '[email protected]', str, 'xmpp_user', 'the secret') \ .and_return('true') fake_userappserver.should_receive('set_cloud_admin_status').with_args( '*****@*****.**', 'true', 'the secret').and_return() fake_userappserver.should_receive('set_capabilities').with_args( '*****@*****.**', UserAppClient.ADMIN_CAPABILITIES, 'the secret').and_return() SOAPpy.should_receive('SOAPProxy').with_args('https://1.2.3.4:4343') \ .and_return(fake_userappserver) # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)