def upgrade(self): """ Allows users to upgrade to the latest version of AppScale.""" contents_as_yaml = yaml.safe_load(self.read_appscalefile()) # Construct the appscale-upgrade command from argv and the contents of # the AppScalefile. command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'verbose' in contents_as_yaml and contents_as_yaml[ 'verbose'] == True: command.append("--verbose") if 'ips_layout' in contents_as_yaml: command.append('--ips_layout') command.append( base64.b64encode(yaml.dump(contents_as_yaml['ips_layout']))) if 'login' in contents_as_yaml: command.extend(['--login', contents_as_yaml['login']]) if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append('--test') options = ParseArgs(command, 'appscale-upgrade').args options.ips = yaml.safe_load(base64.b64decode(options.ips_layout)) options.terminate = False options.clean = False AppScaleTools.upgrade(options)
def testDeployWithCloudAppScalefile(self): # calling 'appscale deploy app' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-upload-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'verbose' : True, 'min' : 1, 'max' : 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call fake_port = 8080 fake_host = 'fake_host' flexmock(AppScaleTools) AppScaleTools.should_receive('upload_app').and_return( (fake_host, fake_port)) app = '/bar/app' (host, port) = appscale.deploy(app) self.assertEquals(fake_host, host) self.assertEquals(fake_port, port)
def up(self): """ Starts an AppScale deployment with the configuration options from the AppScalefile in the current directory. Raises: AppScalefileException: If there is no AppScalefile in the current directory. """ contents = self.read_appscalefile() # If running in a cluster environment, we first need to set up SSH keys contents_as_yaml = yaml.safe_load(contents) if not LocalState.ensure_appscalefile_is_up_to_date(): contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # Construct a run-instances command from the file's contents command = [] for key, value in contents_as_yaml.items(): if key in self.DEPRECATED_ASF_ARGS: raise AppScalefileException( "'{0}' has been deprecated. Refer to {1} to see the full changes.". format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS )) if value is True: command.append(str("--%s" % key)) elif value is False: pass else: if key == "ips_layout": command.append("--ips_layout") command.append(base64.b64encode(yaml.dump(value))) elif key == "disks": command.append("--disks") command.append(base64.b64encode(yaml.dump(value))) elif key == "user_commands": command.append("--user_commands") command.append(base64.b64encode(yaml.dump(value))) else: command.append(str("--%s" % key)) command.append(str("%s" % value)) run_instances_opts = ParseArgs(command, "appscale-run-instances").args if 'infrastructure' not in contents_as_yaml: # Generate a new keypair if necessary. if not self.valid_ssh_key(contents_as_yaml, run_instances_opts): add_keypair_command = [] if 'keyname' in contents_as_yaml: add_keypair_command.append('--keyname') add_keypair_command.append(str(contents_as_yaml['keyname'])) add_keypair_command.append('--ips_layout') add_keypair_command.append( base64.b64encode(yaml.dump(contents_as_yaml['ips_layout']))) add_keypair_opts = ParseArgs( add_keypair_command, 'appscale-add-keypair').args AppScaleTools.add_keypair(add_keypair_opts) AppScaleTools.run_instances(run_instances_opts)
def testDeployWithCloudAppScalefileAndTestFlag(self): # same as before, but with the 'test' flag in our AppScalefile appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'verbose' : True, 'min' : 1, 'max' : 1, 'test' : True } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call fake_port = 8080 fake_host = 'fake_host' flexmock(AppScaleTools) AppScaleTools.should_receive('upload_app').and_return( (fake_host, fake_port)) app = '/bar/app' (host, port) = appscale.deploy(app) self.assertEquals(fake_host, host) self.assertEquals(fake_port, port)
def testDownWithEC2EnvironmentVariables(self): # if the user wants us to use their EC2 credentials when running AppScale, # we should make sure they get set appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'min_machines' : 1, 'max_machines' : 1, 'EC2_ACCESS_KEY' : 'access key', 'EC2_SECRET_KEY' : 'secret key' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-terminate-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('terminate_instances') appscale.down() self.assertEquals('access key', os.environ['EC2_ACCESS_KEY']) self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def set(self, property_name, property_value): """ 'set' provides a cleaner experience for users than the appscale-set-property command, by using the configuration options present in the AppScalefile found in the current working directory. Args: property_name: A str naming the AppController instance variable that should be overwritten. property_value: The new value that should be used for the named property. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # construct the appscale-set-property command command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--property_name") command.append(property_name) command.append("--property_value") command.append(property_value) # and exec it options = ParseArgs(command, "appscale-set-property").args AppScaleTools.set_property(options)
def logs(self, location, other_args=None): """ 'logs' provides a cleaner experience for users than the appscale-gather-logs command, by using the configuration options present in the AppScalefile found in the current working directory. Args: location: The path on the local filesystem where logs should be copied to. other_args: A list of other args from sys.argv. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # construct the appscale-gather-logs command command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--location") command.append(location) if other_args: command += other_args # and exec it options = ParseArgs(command, "appscale-gather-logs").args AppScaleTools.gather_logs(options)
def testUndeployWithCloudAppScalefile(self): # calling 'appscale undeploy app' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-remove-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'verbose' : True, 'min_machines' : 1, 'max_machines' : 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('remove_app') app = 'barapp' appscale.undeploy(app)
def testDownWithEC2EnvironmentVariables(self): # if the user wants us to use their EC2 credentials when running AppScale, # we should make sure they get set appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'min_machines': 1, 'max_machines': 1, 'EC2_ACCESS_KEY': 'access key', 'EC2_SECRET_KEY': 'secret key' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-terminate-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('terminate_instances') appscale.down() self.assertEquals('access key', os.environ['EC2_ACCESS_KEY']) self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def test_all_ok(self): # If the user wants to relocate their app to port X, and nothing else # runs on that port, this should succeed. # Assume that the AppController is running, so is our app, and that other # apps are not running on port 80. fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_app_info_map').with_args( 'the secret').and_return( json.dumps({ self.appid: { 'nginx': 8080 }, 'a-different-app': { 'nginx': 81 } })) fake_appcontroller.should_receive('relocate_app').with_args( self.appid, 80, 443, 'the secret').and_return("OK") flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) rh = flexmock(RemoteHelper) rh.should_receive('sleep_until_port_is_open').and_return() argv = [ '--keyname', self.keyname, '--appname', self.appid, '--http_port', '80', '--https_port', '443' ] options = ParseArgs(argv, self.function).args AppScaleTools.relocate_app(options)
def testUpWithMalformedClusterAppScalefile(self): # if we try to use an IPs layout that isn't a dictionary, we should throw up # and die appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file, with an IPs layout that is a str contents = { 'ips_layout': "'master' 'ip1' 'appengine' 'ip1'", 'keyname': 'boobazblarg', 'group': 'boobazblarg', 'EC2_ACCESS_KEY': '', 'EC2_SECRET_KEY': '' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # finally, mock out the actual appscale tools calls. since we're running # via a cluster, this means we call add-keypair to set up SSH keys, then # run-instances to start appscale flexmock(AppScaleTools) AppScaleTools.should_receive('add_keypair') self.assertRaises(BadConfigurationException, appscale.up)
def upgrade(self): """ Allows users to upgrade to the latest version of AppScale.""" contents_as_yaml = yaml.safe_load(self.read_appscalefile()) # Construct the appscale-upgrade command from argv and the contents of # the AppScalefile. command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True: command.append("--verbose") if 'ips_layout' in contents_as_yaml: command.append('--ips_layout') command.append( base64.b64encode(yaml.dump(contents_as_yaml['ips_layout']))) if 'login' in contents_as_yaml: command.extend(['--login', contents_as_yaml['login']]) if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append('--test') options = ParseArgs(command, 'appscale-upgrade').args options.ips = yaml.safe_load(base64.b64decode(options.ips_layout)) options.terminate = False options.clean = False AppScaleTools.upgrade(options)
def undeploy(self, project_id): """ 'undeploy' is a more accessible way to tell an AppScale deployment to stop hosting a Google App Engine application than 'appscale-remove-app'. It calls that command with the configuration options found in the AppScalefile in the current working directory. Args: project_id: The name of the application that we should remove. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct an remove-app command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'verbose' in contents_as_yaml and contents_as_yaml[ 'verbose'] == True: command.append("--verbose") if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append('--confirm') command.append("--project-id") command.append(project_id) # Finally, exec the command. Don't worry about validating it - # appscale-upload-app will do that for us. options = ParseArgs(command, "appscale-remove-app").args AppScaleTools.remove_app(options)
def testUndeployWithCloudAppScalefile(self): # calling 'appscale undeploy app' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-remove-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min_machines': 1, 'max_machines': 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('remove_app') app = 'barapp' appscale.undeploy(app)
def testDeployWithCloudAppScalefileAndTestFlag(self): # same as before, but with the 'test' flag in our AppScalefile appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min': 1, 'max': 1, 'test': True } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call fake_port = 8080 fake_host = 'fake_host' flexmock(AppScaleTools) AppScaleTools.should_receive('upload_app').and_return( (fake_host, fake_port)) app = '/bar/app' (host, port) = appscale.deploy(app) self.assertEquals(fake_host, host) self.assertEquals(fake_port, port)
def testUpWithMalformedClusterAppScalefile(self): # if we try to use an IPs layout that isn't a dictionary, we should throw up # and die appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file, with an IPs layout that is a str contents = { 'ips_layout': "'master' 'ip1' 'appengine' 'ip1'", 'keyname': 'boobazblarg', 'group' : 'boobazblarg', 'EC2_ACCESS_KEY': '', 'EC2_SECRET_KEY': '' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # finally, mock out the actual appscale tools calls. since we're running # via a cluster, this means we call add-keypair to set up SSH keys, then # run-instances to start appscale flexmock(AppScaleTools) AppScaleTools.should_receive('add_keypair') self.assertRaises(BadConfigurationException, appscale.up)
def undeploy(self, project_id): """ 'undeploy' is a more accessible way to tell an AppScale deployment to stop hosting a Google App Engine application than 'appscale-remove-app'. It calls that command with the configuration options found in the AppScalefile in the current working directory. Args: project_id: The name of the application that we should remove. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct an remove-app command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True: command.append("--verbose") if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append('--confirm') command.append("--project-id") command.append(project_id) # Finally, exec the command. Don't worry about validating it - # appscale-upload-app will do that for us. options = ParseArgs(command, "appscale-remove-app").args AppScaleTools.remove_app(options)
def test_all_ok(self): # If the user wants to relocate their app to port X, and nothing else # runs on that port, this should succeed. # Assume that the AppController is running, so is our app, and that other # apps are not running on port 80. fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_app_info_map').with_args( 'the secret').and_return(json.dumps({ self.appid : { 'nginx' : 8080 }, 'a-different-app' : { 'nginx' : 81 } })) fake_appcontroller.should_receive('relocate_app').with_args(self.appid, 80, 443, 'the secret').and_return("OK") flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) rh = flexmock(RemoteHelper) rh.should_receive('sleep_until_port_is_open').and_return() argv = [ '--keyname', self.keyname, '--appname', self.appid, '--http_port', '80', '--https_port', '443' ] options = ParseArgs(argv, self.function).args AppScaleTools.relocate_app(options)
def testDeployWithCloudAppScalefile(self): # calling 'appscale deploy app' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-upload-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min': 1, 'max': 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-run-instances call fake_port = 8080 fake_host = 'fake_host' flexmock(AppScaleTools) AppScaleTools.should_receive('upload_app').and_return( (fake_host, fake_port)) app = '/bar/app' (host, port) = appscale.deploy(app) self.assertEquals(fake_host, host) self.assertEquals(fake_port, port)
def test_remove_app_and_app_is_running(self): # mock out reading from stdin, and assume the user says 'YES' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('YES') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) app_stats_data = {'apps': {'blargapp': {'http': 8080, 'language': 'python27', 'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}} # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('Database is at public1') fake_appcontroller.should_receive('stop_app').with_args('blargapp', 'the secret').and_return('OK') fake_appcontroller.should_receive('is_app_running').with_args('blargapp', 'the secret').and_return(True).and_return(True).and_return(False) fake_appcontroller.should_receive('does_app_exist').with_args('blargapp', 'the secret').and_return(True) fake_appcontroller.should_receive('get_all_stats').with_args( 'the secret').and_return(json.dumps(app_stats_data)) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps({"node_info": [{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }]})) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) flexmock(RemoteHelper).should_receive('is_port_open').and_return(False) argv = [ "--appname", "blargapp", "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.remove_app(options)
def test_describe_instances_with_two_nodes(self): # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps({ "node_info": [ { "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }, { "public_ip": "public2", "private_ip": "private2", "jobs": ["appengine"] }, ] })) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # assume that there are two machines running in our deployment argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.describe_instances(options)
def down(self, clean=False, terminate=False): """ 'down' provides a nicer experience for users than the appscale-terminate-instances command, by using the configuration options present in the AppScalefile found in the current working directory. Args: clean: A boolean to indicate if the deployment data and metadata needs to be clean. This will clear the datastore. terminate: A boolean to indicate if instances needs to be terminated (valid only if we spawn instances at start). Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct a terminate-instances command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'verbose' in contents_as_yaml and contents_as_yaml[ 'verbose'] == True: command.append("--verbose") if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] command.append("--keyname") command.append(contents_as_yaml['keyname']) else: keyname = 'appscale' if clean: if 'test' not in contents_as_yaml or contents_as_yaml[ 'test'] != True: LocalState.confirm_or_abort( "Clean will delete every data in the deployment.") command.append("--clean") if terminate: infrastructure = LocalState.get_infrastructure(keyname) if infrastructure != "xen" and not LocalState.are_disks_used( keyname) and 'test' not in contents_as_yaml: LocalState.confirm_or_abort( "Terminate will delete instances and the data on them.") command.append("--terminate") if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append("--test") # Finally, exec the command. Don't worry about validating it - # appscale-terminate-instances will do that for us. options = ParseArgs(command, "appscale-terminate-instances").args AppScaleTools.terminate_instances(options) LocalState.cleanup_appscale_files(keyname, terminate) AppScaleLogger.success( "Successfully stopped your AppScale deployment.")
def test_describe_instances_with_two_nodes(self): # mock out writing the secret key to ~/.appscale, as well as reading it # later builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out the SOAP call to the AppController and assume it succeeded fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \ .and_return(json.dumps(['public1', 'public2'])) fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at 1.2.3.4') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \ .and_return(fake_appcontroller) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return(json.dumps( {"node_info": [{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }, { "public_ip": "public2", "private_ip": "private2", "jobs": ["appengine"] }, ]})) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # assume that there are two machines running in our deployment argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.describe_instances(options)
def deploy(self, app, project_id=None): """ 'deploy' is a more accessible way to tell an AppScale deployment to run a Google App Engine application than 'appscale-upload-app'. It calls that command with the configuration options found in the AppScalefile in the current working directory. Args: app: The path (absolute or relative) to the Google App Engine application that should be uploaded. project_id: Which project ID to use to deploy the application. Returns: A tuple containing the host and port where the application is serving traffic from. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct an upload-app command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append("--test") if 'verbose' in contents_as_yaml and contents_as_yaml[ 'verbose'] == True: command.append("--verbose") command.append("--file") command.append(app) if project_id is not None: command.append("--project") command.append(project_id) # Finally, exec the command. Don't worry about validating it - # appscale-upload-app will do that for us. options = ParseArgs(command, "appscale-upload-app").args login_host, http_port = AppScaleTools.upload_app(options) AppScaleTools.update_indexes(options.file, options.keyname, options.project) AppScaleTools.update_cron(options.file, options.keyname, options.project) AppScaleTools.update_queues(options.file, options.keyname, options.project) try: AppScaleTools.update_dispatch(options.file, options.keyname, options.project) except (AdminError, AppScaleException) as e: AppScaleLogger.warn( 'Request to update dispatch failed, if your ' 'dispatch references undeployed services, ignore ' 'this exception: {}'.format(e)) return login_host, http_port
def down(self, clean=False, terminate=False): """ 'down' provides a nicer experience for users than the appscale-terminate-instances command, by using the configuration options present in the AppScalefile found in the current working directory. Args: clean: A boolean to indicate if the deployment data and metadata needs to be clean. This will clear the datastore. terminate: A boolean to indicate if instances needs to be terminated (valid only if we spawn instances at start). Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct a terminate-instances command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True: command.append("--verbose") if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] command.append("--keyname") command.append(contents_as_yaml['keyname']) else: keyname = 'appscale' if clean: if 'test' not in contents_as_yaml or contents_as_yaml['test'] != True: LocalState.confirm_or_abort("Clean will delete every data in the deployment.") command.append("--clean") if terminate: infrastructure = LocalState.get_infrastructure(keyname) if infrastructure != "xen" and not LocalState.are_disks_used( keyname) and 'test' not in contents_as_yaml: LocalState.confirm_or_abort("Terminate will delete instances and the data on them.") command.append("--terminate") if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append("--test") # Finally, exec the command. Don't worry about validating it - # appscale-terminate-instances will do that for us. options = ParseArgs(command, "appscale-terminate-instances").args AppScaleTools.terminate_instances(options) LocalState.cleanup_appscale_files(keyname, terminate) AppScaleLogger.success("Successfully stopped your AppScale deployment.")
def testGetLogsWithKeyname(self): # calling 'appscale logs dir' with a keyname should produce # a command to exec with the --keyname flag appscale = AppScale() contents = {"keyname": "boo"} yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # mock out the actual call to appscale-gather-logs flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') self.assertRaises(BadConfigurationException, appscale.logs, '/baz')
def test_reset_password_for_user_that_exists(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the username and new password from the user builtins.should_receive('raw_input').and_return('*****@*****.**') flexmock(getpass) getpass.should_receive('getpass').and_return('the password') # mock out finding the login node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return( json.dumps({ "node_info": [{ 'public_ip': 'public1', 'private_ip': 'private1', 'jobs': ['login', 'db_master'] }] })) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at public1') fake_appcontroller.should_receive('reset_password').with_args( '*****@*****.**', str, 'the secret').and_return('true') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = ["--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.reset_password(options)
def test_upload_app(self): app_id = 'guestbook' source_path = '{}.tar.gz'.format(app_id) extracted_dir = '/tmp/{}'.format(app_id) head_node = '192.168.33.10' secret = 'secret-key' operation_id = 'operation-1' port = 8080 version_url = 'http://{}:{}'.format(head_node, port) argv = ['--keyname', self.keyname, '--file', source_path, '--test'] options = ParseArgs(argv, self.function).args version = Version('python27', 'app.yaml') version.project_id = app_id flexmock(LocalState).should_receive('extract_tgz_app_to_dir').\ and_return('/tmp/{}'.format(app_id)) flexmock(Version).should_receive('from_tar_gz').and_return(version) flexmock(AppEngineHelper).should_receive('validate_app_id') flexmock(LocalState).should_receive('get_host_with_role').\ and_return(head_node) flexmock(LocalState).should_receive('get_secret_key').and_return( secret) flexmock(RemoteHelper).should_receive('copy_app_to_host').\ with_args(extracted_dir, app_id, self.keyname, {}, None).\ and_return(source_path) flexmock(AdminClient).should_receive('create_version').\ and_return(operation_id) flexmock(AdminClient).should_receive('get_operation').\ and_return({'done': True, 'response': {'versionUrl': version_url}}) flexmock(shutil).should_receive('rmtree').with_args(extracted_dir) flexmock(AppEngineHelper).should_receive('warn_if_version_defined') given_host, given_port = AppScaleTools.upload_app(options) self.assertEquals(given_host, head_node) self.assertEquals(given_port, port) # If provided user is not app admin, deployment should fail. flexmock(AdminClient).should_receive('create_version').\ and_raise(AdminError) self.assertRaises(AdminError, AppScaleTools.upload_app, options) # An application with the PHP runtime should be deployed successfully. version = Version('php', 'app.yaml') version.project_id = app_id flexmock(Version).should_receive('from_tar_gz').and_return(version) flexmock(AdminClient).should_receive('create_version').\ and_return(operation_id) given_host, given_port = AppScaleTools.upload_app(options) self.assertEquals(given_host, head_node) self.assertEquals(given_port, port)
def test_terminate_in_cloud_and_succeeds(self): # Deployment is running on EC2. flexmock(LocalState).should_receive('get_infrastructure').and_return('ec2') # The secret key exists. flexmock(os.path).should_receive('exists').and_return(True) flexmock(RemoteHelper).should_receive('terminate_virtualized_cluster') argv = ['--keyname', self.keyname, '--test'] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_terminate_in_cloud_and_succeeds(self): # Deployment is running on EC2. flexmock(LocalState).should_receive('get_infrastructure').and_return( 'ec2') # The secret key exists. flexmock(os.path).should_receive('exists').and_return(True) flexmock(RemoteHelper).should_receive('terminate_virtualized_cluster') argv = ['--keyname', self.keyname, '--test'] options = ParseArgs(argv, self.function).args AppScaleTools.terminate_instances(options)
def test_upload_app(self): app_id = 'guestbook' source_path = '{}.tar.gz'.format(app_id) extracted_dir = '/tmp/{}'.format(app_id) head_node = '192.168.33.10' secret = 'secret-key' operation_id = 'operation-1' port = 8080 version_url = 'http://{}:{}'.format(head_node, port) argv = ['--keyname', self.keyname, '--file', source_path, '--test'] options = ParseArgs(argv, self.function).args version = Version('python27', 'app.yaml') version.project_id = app_id flexmock(LocalState).should_receive('extract_tgz_app_to_dir').\ and_return('/tmp/{}'.format(app_id)) flexmock(Version).should_receive('from_tar_gz').and_return(version) flexmock(AppEngineHelper).should_receive('validate_app_id') flexmock(LocalState).should_receive('get_host_with_role').\ and_return(head_node) flexmock(LocalState).should_receive('get_secret_key').and_return(secret) flexmock(RemoteHelper).should_receive('copy_app_to_host').\ with_args(extracted_dir, app_id, self.keyname, False, {}, None).\ and_return(source_path) flexmock(AdminClient).should_receive('create_version').\ and_return(operation_id) flexmock(AdminClient).should_receive('get_operation').\ and_return({'done': True, 'response': {'versionUrl': version_url}}) flexmock(shutil).should_receive('rmtree').with_args(extracted_dir) flexmock(AppEngineHelper).should_receive('warn_if_version_defined') given_host, given_port = AppScaleTools.upload_app(options) self.assertEquals(given_host, head_node) self.assertEquals(given_port, port) # If provided user is not app admin, deployment should fail. flexmock(AdminClient).should_receive('create_version').\ and_raise(AdminError) self.assertRaises(AdminError, AppScaleTools.upload_app, options) # An application with the PHP runtime should be deployed successfully. version = Version('php', 'app.yaml') version.project_id = app_id flexmock(Version).should_receive('from_tar_gz').and_return(version) flexmock(AdminClient).should_receive('create_version').\ and_return(operation_id) given_host, given_port = AppScaleTools.upload_app(options) self.assertEquals(given_host, head_node) self.assertEquals(given_port, port)
def test_reset_password_for_user_that_exists(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the username and new password from the user builtins.should_receive('raw_input').and_return('*****@*****.**') flexmock(getpass) getpass.should_receive('getpass').and_return('the password') # mock out finding the login node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return( json.dumps({"node_info": [{ 'public_ip': 'public1', 'private_ip': 'private1', 'jobs': ['login', 'db_master'] }]})) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('status').with_args('the secret') \ .and_return('nothing interesting here') \ .and_return('Database is at not-up-yet') \ .and_return('Database is at public1') fake_appcontroller.should_receive('reset_password').with_args( '*****@*****.**', str, 'the secret').and_return('true') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = [ "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.reset_password(options)
def testGetLogsWithKeyname(self): # calling 'appscale logs dir' with a keyname should produce # a command to exec with the --keyname flag appscale = AppScale() contents = { "keyname" : "boo" } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # mock out the actual call to appscale-gather-logs flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') self.assertRaises(BadConfigurationException, appscale.logs, '/baz')
def test_appscale_in_one_node_virt_deployment_with_login_override(self): # let's say that appscale isn't already running self.local_state.should_receive( 'ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('update_local_metadata').and_return() self.local_state.should_receive('get_local_nodes_info').and_return( json.loads( json.dumps([{ "public_ip": IP_1, "private_ip": IP_1, "roles": ["shadow"] }]))) self.local_state.should_receive('get_secret_key').and_return("fookey") flexmock(RemoteHelper) RemoteHelper.should_receive('enable_root_ssh').and_return() RemoteHelper.should_receive('ensure_machine_is_compatible')\ .and_return() RemoteHelper.should_receive('start_head_node')\ .and_return((IP_1, 'i-ABCDEFG')) RemoteHelper.should_receive('sleep_until_port_is_open').and_return() RemoteHelper.should_receive('copy_local_metadata').and_return() RemoteHelper.should_receive('create_user_accounts').and_return() RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\ .and_return() RemoteHelper.should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) AppControllerClient.should_receive('is_initialized').and_return(True) AppControllerClient.should_receive('set_admin_role').and_return('true') AppControllerClient.should_receive('get_property').\ and_return({'login': IP_1}) flexmock(AppScaleLogger) AppScaleLogger.should_receive('remote_log_tools_state').and_return() # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = ONE_NODE_CLUSTER argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test", "--login_host", "www.booscale.com" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment_with_login_override(self): # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('update_local_metadata').and_return() self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "1.2.3.4", "private_ip" : "1.2.3.4", "jobs" : ["shadow", "login"] }]))) self.local_state.should_receive('get_secret_key').and_return("fookey") flexmock(RemoteHelper) RemoteHelper.should_receive('enable_root_ssh').and_return() RemoteHelper.should_receive('ensure_machine_is_compatible')\ .and_return() RemoteHelper.should_receive('start_head_node')\ .and_return(('1.2.3.4','i-ABCDEFG')) RemoteHelper.should_receive('sleep_until_port_is_open').and_return() RemoteHelper.should_receive('copy_local_metadata').and_return() RemoteHelper.should_receive('create_user_accounts').and_return() RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\ .and_return() RemoteHelper.should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) AppControllerClient.should_receive('is_initialized').and_return(True) AppControllerClient.should_receive('set_admin_role').and_return() # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test", "--login_host", "www.booscale.com" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def testUpWithEC2EnvironmentVariables(self): # if the user wants us to use their EC2 credentials when running AppScale, # we should make sure they get set appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'instance_type': 'm3.medium', 'keyname': 'bookey', 'group': 'boogroup', 'min_machines': 1, 'max_machines': 1, 'EC2_ACCESS_KEY': 'access key', 'EC2_SECRET_KEY': 'secret key', 'zone': 'my-zone-1b' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # finally, pretend that our ec2 zone/image to use exist fake_ec2 = flexmock(name="fake_ec2") fake_ec2.should_receive('get_all_instances') fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \ .and_return('anything') fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \ .and_return() flexmock(boto.ec2) boto.ec2.should_receive('connect_to_region').with_args( 'my-zone-1', aws_access_key_id='access key', aws_secret_access_key='secret key').and_return(fake_ec2) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') appscale.up() self.assertEquals('access key', os.environ['EC2_ACCESS_KEY']) self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def testUpWithCloudAppScalefile(self): # calling 'appscale up' if there is an AppScalefile present # should call appscale-run-instances with the given config # params. here, we assume that the file is intended for use # on EC2 appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'instance_type': 'm3.medium', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'min_machines': 1, 'max_machines': 1, 'zone': 'my-zone-1b' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # throw in some mocks for the argument parsing for credential in EC2Agent.REQUIRED_CREDENTIALS: os.environ[credential] = "baz" # finally, pretend that our ec2 zone and image exists fake_ec2 = flexmock(name="fake_ec2") fake_ec2.should_receive('get_all_instances') fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \ .and_return('anything') fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \ .and_return() flexmock(boto.ec2) boto.ec2.should_receive('connect_to_region').with_args( 'my-zone-1', aws_access_key_id='baz', aws_secret_access_key='baz').and_return(fake_ec2) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') appscale.up()
def testUpWithCloudAppScalefile(self): # calling 'appscale up' if there is an AppScalefile present # should call appscale-run-instances with the given config # params. here, we assume that the file is intended for use # on EC2 appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'instance_type' : 'm3.medium', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'min_machines' : 1, 'max_machines' : 1, 'zone' : 'my-zone-1b' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # throw in some mocks for the argument parsing for credential in EC2Agent.REQUIRED_CREDENTIALS: os.environ[credential] = "baz" # finally, pretend that our ec2 zone and image exists fake_ec2 = flexmock(name="fake_ec2") fake_ec2.should_receive('get_all_instances') fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \ .and_return('anything') fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \ .and_return() flexmock(boto.ec2) boto.ec2.should_receive('connect_to_region').with_args('my-zone-1', aws_access_key_id='baz', aws_secret_access_key='baz').and_return(fake_ec2) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') appscale.up()
def testUpWithEC2EnvironmentVariables(self): # if the user wants us to use their EC2 credentials when running AppScale, # we should make sure they get set appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'instance_type' : 'm3.medium', 'keyname' : 'bookey', 'group' : 'boogroup', 'min_machines' : 1, 'max_machines' : 1, 'EC2_ACCESS_KEY' : 'access key', 'EC2_SECRET_KEY' : 'secret key', 'zone' : 'my-zone-1b' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # finally, pretend that our ec2 zone/image to use exist fake_ec2 = flexmock(name="fake_ec2") fake_ec2.should_receive('get_all_instances') fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \ .and_return('anything') fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \ .and_return() flexmock(boto.ec2) boto.ec2.should_receive('connect_to_region').with_args('my-zone-1', aws_access_key_id='access key', aws_secret_access_key='secret key').and_return(fake_ec2) # finally, mock out the actual appscale-run-instances call flexmock(AppScaleTools) AppScaleTools.should_receive('run_instances') appscale.up() self.assertEquals('access key', os.environ['EC2_ACCESS_KEY']) self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def get(self, property_regex): """ 'get' provides a cleaner experience for users than the appscale-get-property command, by using the configuration options present in the AppScalefile found in the current working directory. Args: property_regex: A regular expression indicating which AppController properties should be retrieved. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # construct the appscale-get-property command command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--property") command.append(property_regex) # and exec it options = ParseArgs(command, "appscale-get-property").args return AppScaleTools.get_property(options)
def deploy(self, app, project_id=None): """ 'deploy' is a more accessible way to tell an AppScale deployment to run a Google App Engine application than 'appscale-upload-app'. It calls that command with the configuration options found in the AppScalefile in the current working directory. Args: app: The path (absolute or relative) to the Google App Engine application that should be uploaded. project_id: Which project ID to use to deploy the application. Returns: A tuple containing the host and port where the application is serving traffic from. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() # Construct an upload-app command from the file's contents command = [] contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) if 'test' in contents_as_yaml and contents_as_yaml['test'] == True: command.append("--test") if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True: command.append("--verbose") command.append("--file") command.append(app) if project_id is not None: command.append("--project") command.append(project_id) # Finally, exec the command. Don't worry about validating it - # appscale-upload-app will do that for us. options = ParseArgs(command, "appscale-upload-app").args login_host, http_port = AppScaleTools.upload_app(options) AppScaleTools.update_cron(options.file, options.keyname) AppScaleTools.update_queues(options.file, options.keyname) return login_host, http_port
def test_remove_app_and_app_is_running(self): # mock out reading from stdin, and assume the user says 'YES' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('YES') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps({ "node_info": [{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }] })) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) flexmock(AdminClient).should_receive('list_services').\ and_return(['default']) flexmock(AdminClient).should_receive('delete_service').\ with_args('blargapp', 'default').and_return('op_id') flexmock(AdminClient).should_receive('get_operation').\ with_args('blargapp', 'op_id').and_return({'done': True}) argv = ["--project-id", "blargapp", "--keyname", self.keyname] options = ParseArgs(argv, self.function).args AppScaleTools.remove_app(options)
def test_remove_app_and_app_is_running(self): # mock out reading from stdin, and assume the user says 'YES' builtins = flexmock(sys.modules['__builtin__']) builtins.should_receive('raw_input').and_return('YES') # mock out reading the secret key builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out reading the locations.json file, and slip in our own json flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps({"node_info": [{ "public_ip": "public1", "private_ip": "private1", "roles": ["shadow", "load_balancer"] }]})) fake_nodes_json.should_receive('write').and_return() builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) flexmock(AdminClient).should_receive('list_services').\ and_return(['default']) flexmock(AdminClient).should_receive('delete_service').\ with_args('blargapp', 'default').and_return('op_id') flexmock(AdminClient).should_receive('get_operation').\ with_args('blargapp', 'op_id').and_return({'done': True}) argv = [ "--project-id", "blargapp", "--keyname", self.keyname ] options = ParseArgs(argv, self.function).args AppScaleTools.remove_app(options)
def testUpWithClusterAppScalefile(self): # calling 'appscale up' if there is an AppScalefile present # should call appscale-run-instances with the given config # params. here, we assume that the file is intended for use # on a virtualized cluster appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'ips_layout': { 'master': 'ip1', 'appengine': 'ip1', 'database': 'ip2', 'zookeeper': 'ip2' }, 'keyname': 'boobazblarg', 'group': 'boobazblarg' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # for this test, let's say that we don't have an SSH key already # set up for ip1 and ip2 # TODO(cgb): Add in tests where we have a key for ip1 but not ip2, # and the case where we have a key but it doesn't work key_path = os.path.expanduser('~/.appscale/boobazblarg.key') os.path.should_receive('exists').with_args(key_path).and_return(False) # finally, mock out the actual appscale tools calls. since we're running # via a cluster, this means we call add-keypair to set up SSH keys, then # run-instances to start appscale flexmock(AppScaleTools) AppScaleTools.should_receive('add_keypair') AppScaleTools.should_receive('run_instances') appscale.up()
def status(self, extra_options_list=None): """ 'status' is a more accessible way to query the state of the AppScale deployment than 'appscale-describe-instances', and calls it with the parameters in the user's AppScalefile. Raises: AppScalefileException: If there is no AppScalefile in the current directory. """ contents = self.read_appscalefile() # Construct a describe-instances command from the file's contents command = extra_options_list or [] contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml['keyname']) # Finally, exec the command. Don't worry about validating it - # appscale-describe-instances will do that for us. options = ParseArgs(command, "appscale-describe-instances").args AppScaleTools.print_cluster_status(options)
def testCreateUserWithAppScalefile(self): # calling 'appscale create-user' with AppScalefile in the local appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min_machines': 1, 'max_machines': 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-create-user call flexmock(AppScaleTools) AppScaleTools.should_receive('create_user') appscale.create_user()
def testUpWithClusterAppScalefile(self): # calling 'appscale up' if there is an AppScalefile present # should call appscale-run-instances with the given config # params. here, we assume that the file is intended for use # on a virtualized cluster appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'ips_layout': {'master': 'ip1', 'appengine': 'ip1', 'database': 'ip2', 'zookeeper': 'ip2'}, 'keyname': 'boobazblarg', 'group': 'boobazblarg' } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) flexmock(os.path) os.path.should_call('exists') os.path.should_receive('exists').with_args( '/boo/' + appscale.APPSCALEFILE).and_return(True) # for this test, let's say that we don't have an SSH key already # set up for ip1 and ip2 # TODO(cgb): Add in tests where we have a key for ip1 but not ip2, # and the case where we have a key but it doesn't work key_path = os.path.expanduser('~/.appscale/boobazblarg.key') os.path.should_receive('exists').with_args(key_path).and_return(False) # finally, mock out the actual appscale tools calls. since we're running # via a cluster, this means we call add-keypair to set up SSH keys, then # run-instances to start appscale flexmock(AppScaleTools) AppScaleTools.should_receive('add_keypair') AppScaleTools.should_receive('run_instances') appscale.up()
def test_get_property(self): # put in a mock for reading the secret file builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') # set the fall-through secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # mock out finding the shadow node's IP address from the json file flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_secret") fake_nodes_json.should_receive('read').and_return( json.dumps({ "node_info": [{ 'public_ip': 'public1', 'private_ip': 'private1', 'jobs': ['login', 'shadow'] }] })) builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out grabbing the userappserver ip from an appcontroller property_name = "name" property_value = "value" fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('set_property').with_args( property_name, property_value, 'the secret').and_return('OK') flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) argv = [ "--keyname", self.keyname, "--property_name", property_name, "--property_value", property_value ] options = ParseArgs(argv, self.function).args result = AppScaleTools.set_property(options) self.assertEqual(None, result)
def relocate(self, appid, http_port, https_port): """ 'relocate' provides a nicer experience for users than the appscale-terminate-instances command, by using the configuration options present in the AppScalefile found in the current working directory. Args: appid: A str indicating the name of the application to relocate. http_port: An int that indicates what port should serve HTTP traffic for this application. https_port: An int that indicates what port should serve HTTPS traffic for this application. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # Construct the appscale-relocate-app command from argv and the contents of # the AppScalefile. command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--appname") command.append(appid) command.append("--http_port") command.append(str(http_port)) command.append("--https_port") command.append(str(https_port)) # and exec it options = ParseArgs(command, "appscale-relocate-app").args AppScaleTools.relocate_app(options)
def testRelocateWithAppScalefile(self): # calling 'appscale relocate' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-relocate-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'verbose' : True, 'min' : 1, 'max' : 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-relocate-app call flexmock(AppScaleTools) AppScaleTools.should_receive('relocate_app') appscale.relocate('myapp', 80, 443)
def testSetPropertyWithAppScalefile(self): # calling 'appscale set' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-get-property' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure' : 'ec2', 'machine' : 'ami-ABCDEFG', 'keyname' : 'bookey', 'group' : 'boogroup', 'verbose' : True, 'min_machines' : 1, 'max_machines' : 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-set-property call flexmock(AppScaleTools) AppScaleTools.should_receive('set_property') appscale.set('key', 'value')
def testRelocateWithAppScalefile(self): # calling 'appscale relocate' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-relocate-app' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min': 1, 'max': 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-relocate-app call flexmock(AppScaleTools) AppScaleTools.should_receive('relocate_app') appscale.relocate('myapp', 80, 443)
def testSetPropertyWithAppScalefile(self): # calling 'appscale set' with an AppScalefile in the local # directory should collect any parameters needed for the # 'appscale-get-property' command and then exec it appscale = AppScale() # Mock out the actual file reading itself, and slip in a YAML-dumped # file contents = { 'infrastructure': 'ec2', 'machine': 'ami-ABCDEFG', 'keyname': 'bookey', 'group': 'boogroup', 'verbose': True, 'min_machines': 1, 'max_machines': 1 } yaml_dumped_contents = yaml.dump(contents) self.addMockForAppScalefile(appscale, yaml_dumped_contents) # finally, mock out the actual appscale-set-property call flexmock(AppScaleTools) AppScaleTools.should_receive('set_property') appscale.set('key', 'value')
def create_user(self, is_admin=False): """ 'create_user' Creates a new user from the tools. Raises: AppScalefileException: If there is no AppScalefile in the current directory. TypeError: If the user does not provide an new user credentials """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) options = ParseArgs(command, "appscale-create-user").args return AppScaleTools.create_user(options, is_admin)