def up(self): """ Starts an AppScale deployment with the configuration options from the AppScalefile in the current directory. Raises: AppScalefileException: If there is no AppScalefile in the current directory. """ contents = self.read_appscalefile() # If running in a cluster environment, we first need to set up SSH keys contents_as_yaml = yaml.safe_load(contents) if not LocalState.ensure_appscalefile_is_up_to_date(): contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # Construct a run-instances command from the file's contents command = [] for key, value in contents_as_yaml.items(): if key in self.DEPRECATED_ASF_ARGS: raise AppScalefileException( "'{0}' has been deprecated. Refer to {1} to see the full changes.". format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS )) if value is True: command.append(str("--%s" % key)) elif value is False: pass else: if key == "ips_layout": command.append("--ips_layout") command.append(base64.b64encode(yaml.dump(value))) elif key == "disks": command.append("--disks") command.append(base64.b64encode(yaml.dump(value))) elif key == "user_commands": command.append("--user_commands") command.append(base64.b64encode(yaml.dump(value))) else: command.append(str("--%s" % key)) command.append(str("%s" % value)) run_instances_opts = ParseArgs(command, "appscale-run-instances").args if 'infrastructure' not in contents_as_yaml: # Generate a new keypair if necessary. if not self.valid_ssh_key(contents_as_yaml, run_instances_opts): add_keypair_command = [] if 'keyname' in contents_as_yaml: add_keypair_command.append('--keyname') add_keypair_command.append(str(contents_as_yaml['keyname'])) add_keypair_command.append('--ips_layout') add_keypair_command.append( base64.b64encode(yaml.dump(contents_as_yaml['ips_layout']))) add_keypair_opts = ParseArgs( add_keypair_command, 'appscale-add-keypair').args AppScaleTools.add_keypair(add_keypair_opts) AppScaleTools.run_instances(run_instances_opts)
def test_appscale_in_one_node_virt_deployment_with_login_override(self): # let's say that appscale isn't already running self.local_state.should_receive( 'ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('update_local_metadata').and_return() self.local_state.should_receive('get_local_nodes_info').and_return( json.loads( json.dumps([{ "public_ip": IP_1, "private_ip": IP_1, "roles": ["shadow"] }]))) self.local_state.should_receive('get_secret_key').and_return("fookey") flexmock(RemoteHelper) RemoteHelper.should_receive('enable_root_ssh').and_return() RemoteHelper.should_receive('ensure_machine_is_compatible')\ .and_return() RemoteHelper.should_receive('start_head_node')\ .and_return((IP_1, 'i-ABCDEFG')) RemoteHelper.should_receive('sleep_until_port_is_open').and_return() RemoteHelper.should_receive('copy_local_metadata').and_return() RemoteHelper.should_receive('create_user_accounts').and_return() RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\ .and_return() RemoteHelper.should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) AppControllerClient.should_receive('is_initialized').and_return(True) AppControllerClient.should_receive('set_admin_role').and_return('true') AppControllerClient.should_receive('get_property').\ and_return({'login': IP_1}) flexmock(AppScaleLogger) AppScaleLogger.should_receive('remote_log_tools_state').and_return() # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = ONE_NODE_CLUSTER argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test", "--login_host", "www.booscale.com" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment_with_login_override(self): # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('update_local_metadata').and_return() self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "1.2.3.4", "private_ip" : "1.2.3.4", "jobs" : ["shadow", "login"] }]))) self.local_state.should_receive('get_secret_key').and_return("fookey") flexmock(RemoteHelper) RemoteHelper.should_receive('enable_root_ssh').and_return() RemoteHelper.should_receive('ensure_machine_is_compatible')\ .and_return() RemoteHelper.should_receive('start_head_node')\ .and_return(('1.2.3.4','i-ABCDEFG')) RemoteHelper.should_receive('sleep_until_port_is_open').and_return() RemoteHelper.should_receive('copy_local_metadata').and_return() RemoteHelper.should_receive('create_user_accounts').and_return() RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\ .and_return() RemoteHelper.should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) AppControllerClient.should_receive('is_initialized').and_return(True) AppControllerClient.should_receive('set_admin_role').and_return() # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test", "--login_host", "www.booscale.com" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \ os.sep + self.keyname + ".key" self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('get_key_path_from_name').and_return( local_appscale_path) # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args('1.23', 'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'], instance_type='m3.medium', count=1, placement='my-zone-1b') # Don't write local metadata files. flexmock(LocalState).should_receive('update_local_metadata') # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo touch /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='mktemp').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('rm -f ') ).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('public1') self.setup_appcontroller_mocks('public1', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }]))) # copying over the locations json file should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/') self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() flexmock(RemoteHelper).should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) # Let's mock the call to describe_instances when checking for old # instances to re-use, and then to start the headnode. pending_instance = flexmock(name='pending_instance', state='pending', key_name=self.keyname, id='i-ABCDEFG') pending_reservation = flexmock(name='pending_reservation', instances=[pending_instance]) no_instances = flexmock(name='no_instances', instances=[]) running_instance = flexmock(name='running_instance', state='running', key_name=self.keyname, id='i-ABCDEFG', ip_address='public1', private_ip_address='private1') running_reservation = flexmock(name='running_reservation', instances=[running_instance]) self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \ .and_return(no_instances) \ .and_return(no_instances).and_return(pending_reservation) \ .and_return(running_reservation) argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--instance_type", "m3.medium", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment(self): self.local_state.should_receive('shell').\ with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet " "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no " "-o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin="cp /root/appscale/AppController/scripts/appcontroller " "/etc/init.d/") self.local_state.should_receive('shell').\ with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet " "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no " "-o UserKnownHostsFile=/dev/null [email protected] ", False, 5, stdin="chmod +x /etc/init.d/appcontroller") self.local_state.should_receive('shell').\ with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet " "-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no " "-o UserKnownHostsFile=/dev/null root@public1 ", False, 5, stdin="cp /root/appscale/AppController/scripts/appcontroller " "/etc/init.d/") # let's say that appscale isn't already running self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() rh = flexmock(RemoteHelper) rh.should_receive('copy_deployment_credentials').and_return() # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').\ with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').\ with_args('logs.appscale.com').and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').\ with_args(secret_key_location, 'r').and_return(fake_secret) self.builtins.should_receive('open').\ with_args(secret_key_location, 'w').and_return(fake_secret) # Don't write local metadata files. flexmock(LocalState).should_receive('update_local_metadata') # mock out copying over the keys self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.key'),False,5) self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell')\ .with_args(re.compile('^openssl'),False,stdin=None)\ .and_return() # mock out removing the old json file self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\ .and_return() # assume that we started monit fine self.local_state.should_receive('shell')\ .with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\ .and_return() # and that we copied over the AppController's monit file self.local_state.should_receive('shell')\ .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\ .and_return() self.local_state.should_receive('shell').\ with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet ' '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no ' '-o UserKnownHostsFile=/dev/null [email protected] ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.setup_socket_mocks('1.2.3.4') self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip": "1.2.3.4", "private_ip": "1.2.3.4", "jobs": ["shadow", "login"] }]))) # Assume the locations files were copied successfully. locations_file = '{}/locations-bookey.yaml'.\ format(RemoteHelper.CONFIG_DIR) self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\ .and_return() locations_json = '{}/locations-bookey.json'.\ format(RemoteHelper.CONFIG_DIR) self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\ .and_return() user_locations = '/root/.appscale/locations-bookey.json' self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\ .and_return() # Assume the secret key was copied successfully. self.local_state.should_receive('shell')\ .with_args(re.compile('^scp .*.secret'), False, 5)\ .and_return() flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) # don't use a 192.168.X.Y IP here, since sometimes we set our virtual # machines to boot with those addresses (and that can mess up our tests). ips_layout = yaml.safe_load(""" master : 1.2.3.4 database: 1.2.3.4 zookeeper: 1.2.3.4 appengine: 1.2.3.4 """) argv = [ "--ips_layout", base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname, "--test" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \ os.sep + self.keyname + ".key" self.local_state.should_receive( 'ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('get_key_path_from_name').and_return( local_appscale_path) # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args( 'POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args( '1.23', 'ami-ABCDEFG', key_name=self.keyname, network_interfaces=None, security_groups=[self.group], instance_type='m3.medium', count=1, placement='my-zone-1b') # Don't write local metadata files. flexmock(LocalState).should_receive('update_local_metadata') # assume that root login is not enabled self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin='ls').and_return( 'Please login as the user "ubuntu" rather than the user "root"' ) # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin='sudo touch /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin='mktemp').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' )).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ') ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin=re.compile('rm -f ')).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), None, 5, stdin=re.compile('rm -rf ')).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args( re.compile('scp .*[r|d]sa'), None, 5).and_return() self.local_state.should_receive('shell').with_args( re.compile('scp .*{0}'.format(self.keyname)), None, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args( re.compile('openssl'), None, stdin=None) self.local_state.should_receive('shell').with_args( re.compile('^ssh'), None, 5, stdin='systemctl start appscale-controller') self.setup_socket_mocks('public1') self.setup_appcontroller_mocks('public1', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return( json.loads( json.dumps([{ "public_ip": "public1", "private_ip": "private1", "roles": ["shadow"] }]))) # copying over the locations json file should be fine self.local_state.should_receive('shell').with_args( re.compile('scp'), None, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args( re.compile('scp'), None, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) flexmock(RemoteHelper).should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) AppControllerClient.should_receive('get_property').\ and_return({'login': '******'}) # Let's mock the call to describe_instances when checking for old # instances to re-use, and then to start the headnode. pending_instance = flexmock(name='pending_instance', state='pending', key_name=self.keyname, id='i-ABCDEFG') pending_reservation = flexmock(name='pending_reservation', instances=[pending_instance]) no_instances = flexmock(name='no_instances', instances=[]) running_instance = flexmock(name='running_instance', state='running', key_name=self.keyname, id='i-ABCDEFG', ip_address='public1', private_ip_address='private1') running_reservation = flexmock(name='running_reservation', instances=[running_instance]) self.fake_ec2.should_receive('get_all_instances') \ .and_return(no_instances) \ .and_return(pending_reservation) \ .and_return(running_reservation) argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--instance_type", "m3.medium", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b", "--EC2_ACCESS_KEY", "baz", "--EC2_SECRET_KEY", "baz" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self): # let's say that appscale isn't already running local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \ os.sep + self.keyname + ".key" self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('get_key_path_from_name').and_return( local_appscale_path) # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args('1.23', 'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'], instance_type='m3.medium', count=1, placement='my-zone-1b') # Don't write local metadata files. flexmock(LocalState).should_receive('update_local_metadata') # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER) # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo touch /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='mktemp').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('rm -f ') ).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}' .format(self.keyname)), False, 5).and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) # and that we copied over the AppController's monit file self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('controller-17443.cfg')) self.setup_socket_mocks('public1') self.setup_appcontroller_mocks('public1', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "public1", "private_ip" : "private1", "jobs" : ["shadow", "login"] }]))) # copying over the locations json file should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/') self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return() flexmock(RemoteHelper).should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--instance_type", "m3.medium", "--machine", "ami-ABCDEFG", "--use_spot_instances", "--max_spot_price", "1.23", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)
def up(self): """ Starts an AppScale deployment with the configuration options from the AppScalefile in the current directory. Raises: AppScalefileException: If there is no AppScalefile in the current directory. """ contents = self.read_appscalefile() # If running in a cluster environment, we first need to set up SSH keys contents_as_yaml = yaml.safe_load(contents) if not LocalState.ensure_appscalefile_is_up_to_date(): contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # Construct a run-instances command from the file's contents command = [] for key, value in contents_as_yaml.items(): if key in self.DEPRECATED_ASF_ARGS: raise AppScalefileException( "'{0}' has been deprecated. Refer to {1} to see the full changes." .format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS)) if value is True: command.append(str("--%s" % key)) elif value is False: pass else: if key == "ips_layout": command.append("--ips_layout") command.append(base64.b64encode(yaml.dump(value))) elif key == "disks": command.append("--disks") command.append(base64.b64encode(yaml.dump(value))) elif key == "user_commands": command.append("--user_commands") command.append(base64.b64encode(yaml.dump(value))) else: command.append(str("--%s" % key)) command.append(str("%s" % value)) run_instances_opts = ParseArgs(command, "appscale-run-instances").args if 'infrastructure' not in contents_as_yaml: # Generate a new keypair if necessary. if not self.valid_ssh_key(contents_as_yaml, run_instances_opts): add_keypair_command = [] if 'keyname' in contents_as_yaml: add_keypair_command.append('--keyname') add_keypair_command.append(str( contents_as_yaml['keyname'])) add_keypair_command.append('--ips_layout') add_keypair_command.append( base64.b64encode(yaml.dump( contents_as_yaml['ips_layout']))) add_keypair_opts = ParseArgs(add_keypair_command, 'appscale-add-keypair').args AppScaleTools.add_keypair(add_keypair_opts) AppScaleTools.run_instances(run_instances_opts)
def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self): # let's say that appscale isn't already running local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \ os.sep + self.keyname + ".key" self.local_state.should_receive('ensure_appscale_isnt_running').and_return() self.local_state.should_receive('make_appscale_directory').and_return() self.local_state.should_receive('get_key_path_from_name').and_return( local_appscale_path) # mock out talking to logs.appscale.com fake_connection = flexmock(name='fake_connection') fake_connection.should_receive('request').with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return() flexmock(httplib) httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \ .and_return(fake_connection) # mock out generating the secret key flexmock(uuid) uuid.should_receive('uuid4').and_return('the secret') # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') fake_secret.should_receive('write').and_return() self.builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) self.builtins.should_receive('open').with_args(secret_key_location, 'w') \ .and_return(fake_secret) self.setup_ec2_mocks() # slip in some fake spot instance info fake_entry = flexmock(name='fake_entry', price=1) self.fake_ec2.should_receive('get_spot_price_history').with_args( start_time=str, end_time=str, product_description='Linux/UNIX', instance_type='m3.medium', availability_zone='my-zone-1b').and_return([fake_entry]) # also mock out acquiring a spot instance self.fake_ec2.should_receive('request_spot_instances').with_args('1.1', 'ami-ABCDEFG', key_name=self.keyname, security_groups=[self.group], instance_type='m3.medium', count=1, placement='my-zone-1b') # Don't write local metadata files. flexmock(LocalState).should_receive('update_local_metadata') # assume that root login is not enabled self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin='ls').and_return( 'Please login as the user "ubuntu" rather than the user "root"') # assume that we can enable root login self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo touch /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin='mktemp').and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile( 'sudo sed -n ' '\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ' ) ).and_return() self.local_state.should_receive('shell').with_args( re.compile('ssh'), False, 5, stdin=re.compile('rm -f ') ).and_return() # and assume that we can copy over our ssh keys fine self.local_state.should_receive('shell').\ with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return() self.local_state.should_receive('shell').\ with_args(re.compile('scp .*{0}'.format(self.keyname)), False, 5).\ and_return() self.local_state.should_receive('shell').\ with_args('ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o ' 'NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no ' '-o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller ' '/etc/init.d/').\ and_return() self.local_state.should_receive('shell').\ with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet ' '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no ' '-o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller ' '/etc/init.d/').\ and_return() self.local_state.should_receive('shell').\ with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet ' '-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no ' '-o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').\ and_return() self.setup_appscale_compatibility_mocks() # mock out generating the private key self.local_state.should_receive('shell').with_args(re.compile('openssl'), False, stdin=None) # assume that we started monit fine self.local_state.should_receive('shell').with_args(re.compile('ssh'), False, 5, stdin=re.compile('monit')) self.local_state.should_receive('shell').with_args( re.compile('^ssh'), False, 5, stdin='service appscale-controller start') self.setup_socket_mocks('elastic-ip') self.setup_appcontroller_mocks('elastic-ip', 'private1') # mock out reading the locations.json file, and slip in our own json self.local_state.should_receive('get_local_nodes_info').and_return(json.loads( json.dumps([{ "public_ip" : "elastic-ip", "private_ip" : "private1", "jobs": ["shadow", "login"] }]))) # copying over the locations yaml and json files should be fine self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('locations-{0}'.format(self.keyname))) # same for the secret key self.local_state.should_receive('shell').with_args(re.compile('scp'), False, 5, stdin=re.compile('{0}.secret'.format(self.keyname))) flexmock(RemoteHelper).should_receive('copy_deployment_credentials') flexmock(AppControllerClient) AppControllerClient.should_receive('does_user_exist').and_return(True) # Let's mock the call to describe_instances when checking for old # instances to re-use, and then to start the headnode. pending_instance = flexmock(name='pending_instance', state='pending', key_name=self.keyname, id='i-ABCDEFG') pending_reservation = flexmock(name='pending_reservation', instances=[pending_instance]) no_instances = flexmock(name='no_instances', instances=[]) running_instance = flexmock(name='running_instance', state='running', key_name=self.keyname, id='i-ABCDEFG', ip_address='public1', private_ip_address='private1') running_reservation = flexmock(name='running_reservation', instances=[running_instance]) self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \ .and_return(no_instances) \ .and_return(no_instances).and_return(pending_reservation) \ .and_return(running_reservation) argv = [ "--min", "1", "--max", "1", "--infrastructure", "ec2", "--machine", "ami-ABCDEFG", "--instance_type", "m3.medium", "--use_spot_instances", "--keyname", self.keyname, "--group", self.group, "--test", "--zone", "my-zone-1b", "--static_ip", "elastic-ip", "--EC2_ACCESS_KEY", "baz", "--EC2_SECRET_KEY", "baz" ] options = ParseArgs(argv, self.function).args AppScaleTools.run_instances(options)