def up(self):
""" Starts an AppScale deployment with the configuration options from the
AppScalefile in the current directory.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
"""
contents = self.read_appscalefile()
# If running in a cluster environment, we first need to set up SSH keys
contents_as_yaml = yaml.safe_load(contents)
if not LocalState.ensure_appscalefile_is_up_to_date():
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# Construct a run-instances command from the file's contents
command = []
for key, value in contents_as_yaml.items():
if key in self.DEPRECATED_ASF_ARGS:
raise AppScalefileException(
"'{0}' has been deprecated. Refer to {1} to see the full changes.".
format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS ))
if value is True:
command.append(str("--%s" % key))
elif value is False:
pass
else:
if key == "ips_layout":
command.append("--ips_layout")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "disks":
command.append("--disks")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "user_commands":
command.append("--user_commands")
command.append(base64.b64encode(yaml.dump(value)))
else:
command.append(str("--%s" % key))
command.append(str("%s" % value))
run_instances_opts = ParseArgs(command, "appscale-run-instances").args
if 'infrastructure' not in contents_as_yaml:
# Generate a new keypair if necessary.
if not self.valid_ssh_key(contents_as_yaml, run_instances_opts):
add_keypair_command = []
if 'keyname' in contents_as_yaml:
add_keypair_command.append('--keyname')
add_keypair_command.append(str(contents_as_yaml['keyname']))
add_keypair_command.append('--ips_layout')
add_keypair_command.append(
base64.b64encode(yaml.dump(contents_as_yaml['ips_layout'])))
add_keypair_opts = ParseArgs(
add_keypair_command, 'appscale-add-keypair').args
AppScaleTools.add_keypair(add_keypair_opts)
AppScaleTools.run_instances(run_instances_opts)
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
self.local_state.should_receive(
'ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('update_local_metadata').and_return()
self.local_state.should_receive('get_local_nodes_info').and_return(
json.loads(
json.dumps([{
"public_ip": IP_1,
"private_ip": IP_1,
"roles": ["shadow"]
}])))
self.local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('enable_root_ssh').and_return()
RemoteHelper.should_receive('ensure_machine_is_compatible')\
.and_return()
RemoteHelper.should_receive('start_head_node')\
.and_return((IP_1, 'i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
RemoteHelper.should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('is_initialized').and_return(True)
AppControllerClient.should_receive('set_admin_role').and_return('true')
AppControllerClient.should_receive('get_property').\
and_return({'login': IP_1})
flexmock(AppScaleLogger)
AppScaleLogger.should_receive('remote_log_tools_state').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = ONE_NODE_CLUSTER
argv = [
"--ips_layout",
base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname,
"--test", "--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('update_local_metadata').and_return()
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "1.2.3.4",
"private_ip" : "1.2.3.4",
"jobs" : ["shadow", "login"]
}])))
self.local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('enable_root_ssh').and_return()
RemoteHelper.should_receive('ensure_machine_is_compatible')\
.and_return()
RemoteHelper.should_receive('start_head_node')\
.and_return(('1.2.3.4','i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
RemoteHelper.should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('is_initialized').and_return(True)
AppControllerClient.should_receive('set_admin_role').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test",
"--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
False, 5).and_return()
self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
.format(self.keyname)), False, 5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('controller-17443.cfg'))
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance', state='pending',
key_name=self.keyname, id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance', state='running',
key_name=self.keyname, id='i-ABCDEFG',
ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
.and_return(no_instances) \
.and_return(no_instances).and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--instance_type", "m3.medium",
"--machine", "ami-ABCDEFG",
"--use_spot_instances",
"--max_spot_price", "1.23",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment(self):
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null [email protected] ",
False, 5, stdin="chmod +x /etc/init.d/appcontroller")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
rh = flexmock(RemoteHelper)
rh.should_receive('copy_deployment_credentials').and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').\
with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').\
with_args('logs.appscale.com').and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').\
with_args(secret_key_location, 'r').and_return(fake_secret)
self.builtins.should_receive('open').\
with_args(secret_key_location, 'w').and_return(fake_secret)
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# mock out copying over the keys
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.key'),False,5)
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell')\
.with_args(re.compile('^openssl'),False,stdin=None)\
.and_return()
# mock out removing the old json file
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume that we started monit fine
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
.and_return()
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null [email protected] ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.setup_socket_mocks('1.2.3.4')
self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip": "1.2.3.4",
"private_ip": "1.2.3.4",
"jobs": ["shadow", "login"]
}])))
# Assume the locations files were copied successfully.
locations_file = '{}/locations-bookey.yaml'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
.and_return()
locations_json = '{}/locations-bookey.json'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
.and_return()
user_locations = '/root/.appscale/locations-bookey.json'
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
.and_return()
# Assume the secret key was copied successfully.
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.secret'), False, 5)\
.and_return()
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive(
'ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args(
'POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args(
'1.23',
'ami-ABCDEFG',
key_name=self.keyname,
network_interfaces=None,
security_groups=[self.group],
instance_type='m3.medium',
count=1,
placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5, stdin='ls').and_return(
'Please login as the user "ubuntu" rather than the user "root"'
)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ')
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5,
stdin=re.compile('rm -f ')).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5,
stdin=re.compile('rm -rf ')).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(
re.compile('scp .*[r|d]sa'), None, 5).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('scp .*{0}'.format(self.keyname)), None,
5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(
re.compile('openssl'), None, stdin=None)
self.local_state.should_receive('shell').with_args(
re.compile('^ssh'),
None,
5,
stdin='systemctl start appscale-controller')
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(
json.loads(
json.dumps([{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(
re.compile('scp'),
None,
5,
stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(
re.compile('scp'),
None,
5,
stdin=re.compile('{0}.secret'.format(self.keyname)))
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('get_property').\
and_return({'login': '******'})
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance',
state='pending',
key_name=self.keyname,
id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance',
state='running',
key_name=self.keyname,
id='i-ABCDEFG',
ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances') \
.and_return(no_instances) \
.and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1", "--max", "1", "--infrastructure", "ec2",
"--instance_type", "m3.medium", "--machine", "ami-ABCDEFG",
"--use_spot_instances", "--max_spot_price", "1.23", "--keyname",
self.keyname, "--group", self.group, "--test", "--zone",
"my-zone-1b", "--EC2_ACCESS_KEY", "baz", "--EC2_SECRET_KEY", "baz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
False, 5).and_return()
self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
.format(self.keyname)), False, 5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('controller-17443.cfg'))
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--instance_type", "m3.medium",
"--machine", "ami-ABCDEFG",
"--use_spot_instances",
"--max_spot_price", "1.23",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment(self):
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null [email protected] ",
False, 5, stdin="chmod +x /etc/init.d/appcontroller")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
rh = flexmock(RemoteHelper)
rh.should_receive('copy_deployment_credentials').and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').\
with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').\
with_args('logs.appscale.com').and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').\
with_args(secret_key_location, 'r').and_return(fake_secret)
self.builtins.should_receive('open').\
with_args(secret_key_location, 'w').and_return(fake_secret)
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# mock out copying over the keys
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.key'),False,5)
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell')\
.with_args(re.compile('^openssl'),False,stdin=None)\
.and_return()
# mock out removing the old json file
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume that we started monit fine
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
.and_return()
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null [email protected] ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.setup_socket_mocks('1.2.3.4')
self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip": "1.2.3.4",
"private_ip": "1.2.3.4",
"jobs": ["shadow", "login"]
}])))
# Assume the locations files were copied successfully.
locations_file = '{}/locations-bookey.yaml'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
.and_return()
locations_json = '{}/locations-bookey.json'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
.and_return()
user_locations = '/root/.appscale/locations-bookey.json'
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
.and_return()
# Assume the secret key was copied successfully.
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.secret'), False, 5)\
.and_return()
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def up(self):
""" Starts an AppScale deployment with the configuration options from the
AppScalefile in the current directory.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
"""
contents = self.read_appscalefile()
# If running in a cluster environment, we first need to set up SSH keys
contents_as_yaml = yaml.safe_load(contents)
if not LocalState.ensure_appscalefile_is_up_to_date():
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# Construct a run-instances command from the file's contents
command = []
for key, value in contents_as_yaml.items():
if key in self.DEPRECATED_ASF_ARGS:
raise AppScalefileException(
"'{0}' has been deprecated. Refer to {1} to see the full changes."
.format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS))
if value is True:
command.append(str("--%s" % key))
elif value is False:
pass
else:
if key == "ips_layout":
command.append("--ips_layout")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "disks":
command.append("--disks")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "user_commands":
command.append("--user_commands")
command.append(base64.b64encode(yaml.dump(value)))
else:
command.append(str("--%s" % key))
command.append(str("%s" % value))
run_instances_opts = ParseArgs(command, "appscale-run-instances").args
if 'infrastructure' not in contents_as_yaml:
# Generate a new keypair if necessary.
if not self.valid_ssh_key(contents_as_yaml, run_instances_opts):
add_keypair_command = []
if 'keyname' in contents_as_yaml:
add_keypair_command.append('--keyname')
add_keypair_command.append(str(
contents_as_yaml['keyname']))
add_keypair_command.append('--ips_layout')
add_keypair_command.append(
base64.b64encode(yaml.dump(
contents_as_yaml['ips_layout'])))
add_keypair_opts = ParseArgs(add_keypair_command,
'appscale-add-keypair').args
AppScaleTools.add_keypair(add_keypair_opts)
AppScaleTools.run_instances(run_instances_opts)
def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# slip in some fake spot instance info
fake_entry = flexmock(name='fake_entry', price=1)
self.fake_ec2.should_receive('get_spot_price_history').with_args(
start_time=str, end_time=str,
product_description='Linux/UNIX', instance_type='m3.medium',
availability_zone='my-zone-1b').and_return([fake_entry])
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.1',
'ami-ABCDEFG', key_name=self.keyname, security_groups=[self.group],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(
'Please login as the user "ubuntu" rather than the user "root"')
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').\
with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return()
self.local_state.should_receive('shell').\
with_args(re.compile('scp .*{0}'.format(self.keyname)), False, 5).\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o '
'NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@public1 ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller '
'/etc/init.d/').\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@elastic-ip ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller '
'/etc/init.d/').\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@elastic-ip ',
False, 5, stdin='chmod +x /etc/init.d/appcontroller').\
and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
self.local_state.should_receive('shell').with_args(
re.compile('^ssh'), False, 5, stdin='service appscale-controller start')
self.setup_socket_mocks('elastic-ip')
self.setup_appcontroller_mocks('elastic-ip', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "elastic-ip",
"private_ip" : "private1",
"jobs": ["shadow", "login"]
}])))
# copying over the locations yaml and json files should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance', state='pending',
key_name=self.keyname, id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance', state='running',
key_name=self.keyname, id='i-ABCDEFG', ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
.and_return(no_instances) \
.and_return(no_instances).and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--machine", "ami-ABCDEFG",
"--instance_type", "m3.medium",
"--use_spot_instances",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b",
"--static_ip", "elastic-ip",
"--EC2_ACCESS_KEY", "baz",
"--EC2_SECRET_KEY", "baz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
