def upgrade(self):
""" Allows users to upgrade to the latest version of AppScale."""
contents_as_yaml = yaml.safe_load(self.read_appscalefile())
# Construct the appscale-upgrade command from argv and the contents of
# the AppScalefile.
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
command.append("--verbose")
if 'ips_layout' in contents_as_yaml:
command.append('--ips_layout')
command.append(
base64.b64encode(yaml.dump(contents_as_yaml['ips_layout'])))
if 'login' in contents_as_yaml:
command.extend(['--login', contents_as_yaml['login']])
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append('--test')
options = ParseArgs(command, 'appscale-upgrade').args
options.ips = yaml.safe_load(base64.b64decode(options.ips_layout))
options.terminate = False
options.clean = False
AppScaleTools.upgrade(options)
def testDeployWithCloudAppScalefile(self):
# calling 'appscale deploy app' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-upload-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'verbose' : True,
'min' : 1,
'max' : 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
fake_port = 8080
fake_host = 'fake_host'
flexmock(AppScaleTools)
AppScaleTools.should_receive('upload_app').and_return(
(fake_host, fake_port))
app = '/bar/app'
(host, port) = appscale.deploy(app)
self.assertEquals(fake_host, host)
self.assertEquals(fake_port, port)
def up(self):
""" Starts an AppScale deployment with the configuration options from the
AppScalefile in the current directory.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
"""
contents = self.read_appscalefile()
# If running in a cluster environment, we first need to set up SSH keys
contents_as_yaml = yaml.safe_load(contents)
if not LocalState.ensure_appscalefile_is_up_to_date():
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# Construct a run-instances command from the file's contents
command = []
for key, value in contents_as_yaml.items():
if key in self.DEPRECATED_ASF_ARGS:
raise AppScalefileException(
"'{0}' has been deprecated. Refer to {1} to see the full changes.".
format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS ))
if value is True:
command.append(str("--%s" % key))
elif value is False:
pass
else:
if key == "ips_layout":
command.append("--ips_layout")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "disks":
command.append("--disks")
command.append(base64.b64encode(yaml.dump(value)))
elif key == "user_commands":
command.append("--user_commands")
command.append(base64.b64encode(yaml.dump(value)))
else:
command.append(str("--%s" % key))
command.append(str("%s" % value))
run_instances_opts = ParseArgs(command, "appscale-run-instances").args
if 'infrastructure' not in contents_as_yaml:
# Generate a new keypair if necessary.
if not self.valid_ssh_key(contents_as_yaml, run_instances_opts):
add_keypair_command = []
if 'keyname' in contents_as_yaml:
add_keypair_command.append('--keyname')
add_keypair_command.append(str(contents_as_yaml['keyname']))
add_keypair_command.append('--ips_layout')
add_keypair_command.append(
base64.b64encode(yaml.dump(contents_as_yaml['ips_layout'])))
add_keypair_opts = ParseArgs(
add_keypair_command, 'appscale-add-keypair').args
AppScaleTools.add_keypair(add_keypair_opts)
AppScaleTools.run_instances(run_instances_opts)
def testDeployWithCloudAppScalefileAndTestFlag(self):
# same as before, but with the 'test' flag in our AppScalefile
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'verbose' : True,
'min' : 1,
'max' : 1,
'test' : True
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
fake_port = 8080
fake_host = 'fake_host'
flexmock(AppScaleTools)
AppScaleTools.should_receive('upload_app').and_return(
(fake_host, fake_port))
app = '/bar/app'
(host, port) = appscale.deploy(app)
self.assertEquals(fake_host, host)
self.assertEquals(fake_port, port)
def testDownWithEC2EnvironmentVariables(self):
# if the user wants us to use their EC2 credentials when running AppScale,
# we should make sure they get set
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'min_machines' : 1,
'max_machines' : 1,
'EC2_ACCESS_KEY' : 'access key',
'EC2_SECRET_KEY' : 'secret key'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-terminate-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('terminate_instances')
appscale.down()
self.assertEquals('access key', os.environ['EC2_ACCESS_KEY'])
self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def set(self, property_name, property_value):
""" 'set' provides a cleaner experience for users than the
appscale-set-property command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
property_name: A str naming the AppController instance variable that
should be overwritten.
property_value: The new value that should be used for the named property.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-set-property command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--property_name")
command.append(property_name)
command.append("--property_value")
command.append(property_value)
# and exec it
options = ParseArgs(command, "appscale-set-property").args
AppScaleTools.set_property(options)
def logs(self, location, other_args=None):
""" 'logs' provides a cleaner experience for users than the
appscale-gather-logs command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
location: The path on the local filesystem where logs should be copied to.
other_args: A list of other args from sys.argv.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-gather-logs command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--location")
command.append(location)
if other_args:
command += other_args
# and exec it
options = ParseArgs(command, "appscale-gather-logs").args
AppScaleTools.gather_logs(options)
def testUndeployWithCloudAppScalefile(self):
# calling 'appscale undeploy app' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-remove-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'verbose' : True,
'min_machines' : 1,
'max_machines' : 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('remove_app')
app = 'barapp'
appscale.undeploy(app)
def testDownWithEC2EnvironmentVariables(self):
# if the user wants us to use their EC2 credentials when running AppScale,
# we should make sure they get set
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'min_machines': 1,
'max_machines': 1,
'EC2_ACCESS_KEY': 'access key',
'EC2_SECRET_KEY': 'secret key'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-terminate-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('terminate_instances')
appscale.down()
self.assertEquals('access key', os.environ['EC2_ACCESS_KEY'])
self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def test_all_ok(self):
# If the user wants to relocate their app to port X, and nothing else
# runs on that port, this should succeed.
# Assume that the AppController is running, so is our app, and that other
# apps are not running on port 80.
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_app_info_map').with_args(
'the secret').and_return(
json.dumps({
self.appid: {
'nginx': 8080
},
'a-different-app': {
'nginx': 81
}
}))
fake_appcontroller.should_receive('relocate_app').with_args(
self.appid, 80, 443, 'the secret').and_return("OK")
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
rh = flexmock(RemoteHelper)
rh.should_receive('sleep_until_port_is_open').and_return()
argv = [
'--keyname', self.keyname, '--appname', self.appid, '--http_port',
'80', '--https_port', '443'
]
options = ParseArgs(argv, self.function).args
AppScaleTools.relocate_app(options)
def testUpWithMalformedClusterAppScalefile(self):
# if we try to use an IPs layout that isn't a dictionary, we should throw up
# and die
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file, with an IPs layout that is a str
contents = {
'ips_layout': "'master' 'ip1' 'appengine' 'ip1'",
'keyname': 'boobazblarg',
'group': 'boobazblarg',
'EC2_ACCESS_KEY': '',
'EC2_SECRET_KEY': ''
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# finally, mock out the actual appscale tools calls. since we're running
# via a cluster, this means we call add-keypair to set up SSH keys, then
# run-instances to start appscale
flexmock(AppScaleTools)
AppScaleTools.should_receive('add_keypair')
self.assertRaises(BadConfigurationException, appscale.up)
def upgrade(self):
""" Allows users to upgrade to the latest version of AppScale."""
contents_as_yaml = yaml.safe_load(self.read_appscalefile())
# Construct the appscale-upgrade command from argv and the contents of
# the AppScalefile.
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
command.append("--verbose")
if 'ips_layout' in contents_as_yaml:
command.append('--ips_layout')
command.append(
base64.b64encode(yaml.dump(contents_as_yaml['ips_layout'])))
if 'login' in contents_as_yaml:
command.extend(['--login', contents_as_yaml['login']])
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append('--test')
options = ParseArgs(command, 'appscale-upgrade').args
options.ips = yaml.safe_load(base64.b64decode(options.ips_layout))
options.terminate = False
options.clean = False
AppScaleTools.upgrade(options)
def undeploy(self, project_id):
""" 'undeploy' is a more accessible way to tell an AppScale deployment to
stop hosting a Google App Engine application than 'appscale-remove-app'. It
calls that command with the configuration options found in the AppScalefile
in the current working directory.
Args:
project_id: The name of the application that we should remove.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct an remove-app command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
command.append("--verbose")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append('--confirm')
command.append("--project-id")
command.append(project_id)
# Finally, exec the command. Don't worry about validating it -
# appscale-upload-app will do that for us.
options = ParseArgs(command, "appscale-remove-app").args
AppScaleTools.remove_app(options)
def testUndeployWithCloudAppScalefile(self):
# calling 'appscale undeploy app' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-remove-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min_machines': 1,
'max_machines': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('remove_app')
app = 'barapp'
appscale.undeploy(app)
def testDeployWithCloudAppScalefileAndTestFlag(self):
# same as before, but with the 'test' flag in our AppScalefile
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min': 1,
'max': 1,
'test': True
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
fake_port = 8080
fake_host = 'fake_host'
flexmock(AppScaleTools)
AppScaleTools.should_receive('upload_app').and_return(
(fake_host, fake_port))
app = '/bar/app'
(host, port) = appscale.deploy(app)
self.assertEquals(fake_host, host)
self.assertEquals(fake_port, port)
def testUpWithMalformedClusterAppScalefile(self):
# if we try to use an IPs layout that isn't a dictionary, we should throw up
# and die
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file, with an IPs layout that is a str
contents = {
'ips_layout': "'master' 'ip1' 'appengine' 'ip1'",
'keyname': 'boobazblarg', 'group' : 'boobazblarg', 'EC2_ACCESS_KEY': '',
'EC2_SECRET_KEY': ''
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# finally, mock out the actual appscale tools calls. since we're running
# via a cluster, this means we call add-keypair to set up SSH keys, then
# run-instances to start appscale
flexmock(AppScaleTools)
AppScaleTools.should_receive('add_keypair')
self.assertRaises(BadConfigurationException, appscale.up)
def set(self, property_name, property_value):
""" 'set' provides a cleaner experience for users than the
appscale-set-property command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
property_name: A str naming the AppController instance variable that
should be overwritten.
property_value: The new value that should be used for the named property.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-set-property command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--property_name")
command.append(property_name)
command.append("--property_value")
command.append(property_value)
# and exec it
options = ParseArgs(command, "appscale-set-property").args
AppScaleTools.set_property(options)
def undeploy(self, project_id):
""" 'undeploy' is a more accessible way to tell an AppScale deployment to
stop hosting a Google App Engine application than 'appscale-remove-app'. It
calls that command with the configuration options found in the AppScalefile
in the current working directory.
Args:
project_id: The name of the application that we should remove.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct an remove-app command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
command.append("--verbose")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append('--confirm')
command.append("--project-id")
command.append(project_id)
# Finally, exec the command. Don't worry about validating it -
# appscale-upload-app will do that for us.
options = ParseArgs(command, "appscale-remove-app").args
AppScaleTools.remove_app(options)
def logs(self, location, other_args=None):
""" 'logs' provides a cleaner experience for users than the
appscale-gather-logs command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
location: The path on the local filesystem where logs should be copied to.
other_args: A list of other args from sys.argv.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-gather-logs command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--location")
command.append(location)
if other_args:
command += other_args
# and exec it
options = ParseArgs(command, "appscale-gather-logs").args
AppScaleTools.gather_logs(options)
def test_all_ok(self):
# If the user wants to relocate their app to port X, and nothing else
# runs on that port, this should succeed.
# Assume that the AppController is running, so is our app, and that other
# apps are not running on port 80.
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_app_info_map').with_args(
'the secret').and_return(json.dumps({
self.appid : {
'nginx' : 8080
},
'a-different-app' : {
'nginx' : 81
}
}))
fake_appcontroller.should_receive('relocate_app').with_args(self.appid, 80,
443, 'the secret').and_return("OK")
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
rh = flexmock(RemoteHelper)
rh.should_receive('sleep_until_port_is_open').and_return()
argv = [
'--keyname', self.keyname,
'--appname', self.appid,
'--http_port', '80',
'--https_port', '443'
]
options = ParseArgs(argv, self.function).args
AppScaleTools.relocate_app(options)
def testDeployWithCloudAppScalefile(self):
# calling 'appscale deploy app' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-upload-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min': 1,
'max': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-run-instances call
fake_port = 8080
fake_host = 'fake_host'
flexmock(AppScaleTools)
AppScaleTools.should_receive('upload_app').and_return(
(fake_host, fake_port))
app = '/bar/app'
(host, port) = appscale.deploy(app)
self.assertEquals(fake_host, host)
self.assertEquals(fake_port, port)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
app_stats_data = {'apps': {'blargapp': {'http': 8080, 'language': 'python27',
'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('stop_app').with_args('blargapp',
'the secret').and_return('OK')
fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
'the secret').and_return(True).and_return(True).and_return(False)
fake_appcontroller.should_receive('does_app_exist').with_args('blargapp',
'the secret').and_return(True)
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(RemoteHelper).should_receive('is_port_open').and_return(False)
argv = [
"--appname", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [
{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["appengine"]
},
]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def down(self, clean=False, terminate=False):
""" 'down' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
clean: A boolean to indicate if the deployment data and metadata
needs to be clean. This will clear the datastore.
terminate: A boolean to indicate if instances needs to be terminated
(valid only if we spawn instances at start).
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct a terminate-instances command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
command.append("--verbose")
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
else:
keyname = 'appscale'
if clean:
if 'test' not in contents_as_yaml or contents_as_yaml[
'test'] != True:
LocalState.confirm_or_abort(
"Clean will delete every data in the deployment.")
command.append("--clean")
if terminate:
infrastructure = LocalState.get_infrastructure(keyname)
if infrastructure != "xen" and not LocalState.are_disks_used(
keyname) and 'test' not in contents_as_yaml:
LocalState.confirm_or_abort(
"Terminate will delete instances and the data on them.")
command.append("--terminate")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
# Finally, exec the command. Don't worry about validating it -
# appscale-terminate-instances will do that for us.
options = ParseArgs(command, "appscale-terminate-instances").args
AppScaleTools.terminate_instances(options)
LocalState.cleanup_appscale_files(keyname, terminate)
AppScaleLogger.success(
"Successfully stopped your AppScale deployment.")
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps(
{"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["appengine"]
},
]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def deploy(self, app, project_id=None):
""" 'deploy' is a more accessible way to tell an AppScale deployment to run a
Google App Engine application than 'appscale-upload-app'. It calls that
command with the configuration options found in the AppScalefile in the
current working directory.
Args:
app: The path (absolute or relative) to the Google App Engine application
that should be uploaded.
project_id: Which project ID to use to deploy the application.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct an upload-app command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
command.append("--verbose")
command.append("--file")
command.append(app)
if project_id is not None:
command.append("--project")
command.append(project_id)
# Finally, exec the command. Don't worry about validating it -
# appscale-upload-app will do that for us.
options = ParseArgs(command, "appscale-upload-app").args
login_host, http_port = AppScaleTools.upload_app(options)
AppScaleTools.update_indexes(options.file, options.keyname,
options.project)
AppScaleTools.update_cron(options.file, options.keyname,
options.project)
AppScaleTools.update_queues(options.file, options.keyname,
options.project)
try:
AppScaleTools.update_dispatch(options.file, options.keyname,
options.project)
except (AdminError, AppScaleException) as e:
AppScaleLogger.warn(
'Request to update dispatch failed, if your '
'dispatch references undeployed services, ignore '
'this exception: {}'.format(e))
return login_host, http_port
def down(self, clean=False, terminate=False):
""" 'down' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
clean: A boolean to indicate if the deployment data and metadata
needs to be clean. This will clear the datastore.
terminate: A boolean to indicate if instances needs to be terminated
(valid only if we spawn instances at start).
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct a terminate-instances command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
command.append("--verbose")
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
else:
keyname = 'appscale'
if clean:
if 'test' not in contents_as_yaml or contents_as_yaml['test'] != True:
LocalState.confirm_or_abort("Clean will delete every data in the deployment.")
command.append("--clean")
if terminate:
infrastructure = LocalState.get_infrastructure(keyname)
if infrastructure != "xen" and not LocalState.are_disks_used(
keyname) and 'test' not in contents_as_yaml:
LocalState.confirm_or_abort("Terminate will delete instances and the data on them.")
command.append("--terminate")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
# Finally, exec the command. Don't worry about validating it -
# appscale-terminate-instances will do that for us.
options = ParseArgs(command, "appscale-terminate-instances").args
AppScaleTools.terminate_instances(options)
LocalState.cleanup_appscale_files(keyname, terminate)
AppScaleLogger.success("Successfully stopped your AppScale deployment.")
def testGetLogsWithKeyname(self):
# calling 'appscale logs dir' with a keyname should produce
# a command to exec with the --keyname flag
appscale = AppScale()
contents = {"keyname": "boo"}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# mock out the actual call to appscale-gather-logs
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
self.assertRaises(BadConfigurationException, appscale.logs, '/baz')
def test_reset_password_for_user_that_exists(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'db_master']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('reset_password').with_args(
'*****@*****.**', str, 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.reset_password(options)
def test_upload_app(self):
app_id = 'guestbook'
source_path = '{}.tar.gz'.format(app_id)
extracted_dir = '/tmp/{}'.format(app_id)
head_node = '192.168.33.10'
secret = 'secret-key'
operation_id = 'operation-1'
port = 8080
version_url = 'http://{}:{}'.format(head_node, port)
argv = ['--keyname', self.keyname, '--file', source_path, '--test']
options = ParseArgs(argv, self.function).args
version = Version('python27', 'app.yaml')
version.project_id = app_id
flexmock(LocalState).should_receive('extract_tgz_app_to_dir').\
and_return('/tmp/{}'.format(app_id))
flexmock(Version).should_receive('from_tar_gz').and_return(version)
flexmock(AppEngineHelper).should_receive('validate_app_id')
flexmock(LocalState).should_receive('get_host_with_role').\
and_return(head_node)
flexmock(LocalState).should_receive('get_secret_key').and_return(
secret)
flexmock(RemoteHelper).should_receive('copy_app_to_host').\
with_args(extracted_dir, app_id, self.keyname, {}, None).\
and_return(source_path)
flexmock(AdminClient).should_receive('create_version').\
and_return(operation_id)
flexmock(AdminClient).should_receive('get_operation').\
and_return({'done': True, 'response': {'versionUrl': version_url}})
flexmock(shutil).should_receive('rmtree').with_args(extracted_dir)
flexmock(AppEngineHelper).should_receive('warn_if_version_defined')
given_host, given_port = AppScaleTools.upload_app(options)
self.assertEquals(given_host, head_node)
self.assertEquals(given_port, port)
# If provided user is not app admin, deployment should fail.
flexmock(AdminClient).should_receive('create_version').\
and_raise(AdminError)
self.assertRaises(AdminError, AppScaleTools.upload_app, options)
# An application with the PHP runtime should be deployed successfully.
version = Version('php', 'app.yaml')
version.project_id = app_id
flexmock(Version).should_receive('from_tar_gz').and_return(version)
flexmock(AdminClient).should_receive('create_version').\
and_return(operation_id)
given_host, given_port = AppScaleTools.upload_app(options)
self.assertEquals(given_host, head_node)
self.assertEquals(given_port, port)
def test_terminate_in_cloud_and_succeeds(self):
# Deployment is running on EC2.
flexmock(LocalState).should_receive('get_infrastructure').and_return('ec2')
# The secret key exists.
flexmock(os.path).should_receive('exists').and_return(True)
flexmock(RemoteHelper).should_receive('terminate_virtualized_cluster')
argv = ['--keyname', self.keyname,
'--test']
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_terminate_in_cloud_and_succeeds(self):
# Deployment is running on EC2.
flexmock(LocalState).should_receive('get_infrastructure').and_return(
'ec2')
# The secret key exists.
flexmock(os.path).should_receive('exists').and_return(True)
flexmock(RemoteHelper).should_receive('terminate_virtualized_cluster')
argv = ['--keyname', self.keyname, '--test']
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_upload_app(self):
app_id = 'guestbook'
source_path = '{}.tar.gz'.format(app_id)
extracted_dir = '/tmp/{}'.format(app_id)
head_node = '192.168.33.10'
secret = 'secret-key'
operation_id = 'operation-1'
port = 8080
version_url = 'http://{}:{}'.format(head_node, port)
argv = ['--keyname', self.keyname, '--file', source_path, '--test']
options = ParseArgs(argv, self.function).args
version = Version('python27', 'app.yaml')
version.project_id = app_id
flexmock(LocalState).should_receive('extract_tgz_app_to_dir').\
and_return('/tmp/{}'.format(app_id))
flexmock(Version).should_receive('from_tar_gz').and_return(version)
flexmock(AppEngineHelper).should_receive('validate_app_id')
flexmock(LocalState).should_receive('get_host_with_role').\
and_return(head_node)
flexmock(LocalState).should_receive('get_secret_key').and_return(secret)
flexmock(RemoteHelper).should_receive('copy_app_to_host').\
with_args(extracted_dir, app_id, self.keyname, False, {}, None).\
and_return(source_path)
flexmock(AdminClient).should_receive('create_version').\
and_return(operation_id)
flexmock(AdminClient).should_receive('get_operation').\
and_return({'done': True, 'response': {'versionUrl': version_url}})
flexmock(shutil).should_receive('rmtree').with_args(extracted_dir)
flexmock(AppEngineHelper).should_receive('warn_if_version_defined')
given_host, given_port = AppScaleTools.upload_app(options)
self.assertEquals(given_host, head_node)
self.assertEquals(given_port, port)
# If provided user is not app admin, deployment should fail.
flexmock(AdminClient).should_receive('create_version').\
and_raise(AdminError)
self.assertRaises(AdminError, AppScaleTools.upload_app, options)
# An application with the PHP runtime should be deployed successfully.
version = Version('php', 'app.yaml')
version.project_id = app_id
flexmock(Version).should_receive('from_tar_gz').and_return(version)
flexmock(AdminClient).should_receive('create_version').\
and_return(operation_id)
given_host, given_port = AppScaleTools.upload_app(options)
self.assertEquals(given_host, head_node)
self.assertEquals(given_port, port)
def test_reset_password_for_user_that_exists(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'db_master']
}]}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('reset_password').with_args(
'*****@*****.**', str, 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.reset_password(options)
def testGetLogsWithKeyname(self):
# calling 'appscale logs dir' with a keyname should produce
# a command to exec with the --keyname flag
appscale = AppScale()
contents = {
"keyname" : "boo"
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# mock out the actual call to appscale-gather-logs
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
self.assertRaises(BadConfigurationException, appscale.logs, '/baz')
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
self.local_state.should_receive(
'ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('update_local_metadata').and_return()
self.local_state.should_receive('get_local_nodes_info').and_return(
json.loads(
json.dumps([{
"public_ip": IP_1,
"private_ip": IP_1,
"roles": ["shadow"]
}])))
self.local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('enable_root_ssh').and_return()
RemoteHelper.should_receive('ensure_machine_is_compatible')\
.and_return()
RemoteHelper.should_receive('start_head_node')\
.and_return((IP_1, 'i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
RemoteHelper.should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('is_initialized').and_return(True)
AppControllerClient.should_receive('set_admin_role').and_return('true')
AppControllerClient.should_receive('get_property').\
and_return({'login': IP_1})
flexmock(AppScaleLogger)
AppScaleLogger.should_receive('remote_log_tools_state').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = ONE_NODE_CLUSTER
argv = [
"--ips_layout",
base64.b64encode(yaml.dump(ips_layout)), "--keyname", self.keyname,
"--test", "--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('update_local_metadata').and_return()
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "1.2.3.4",
"private_ip" : "1.2.3.4",
"jobs" : ["shadow", "login"]
}])))
self.local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('enable_root_ssh').and_return()
RemoteHelper.should_receive('ensure_machine_is_compatible')\
.and_return()
RemoteHelper.should_receive('start_head_node')\
.and_return(('1.2.3.4','i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
RemoteHelper.should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('is_initialized').and_return(True)
AppControllerClient.should_receive('set_admin_role').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test",
"--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def testUpWithEC2EnvironmentVariables(self):
# if the user wants us to use their EC2 credentials when running AppScale,
# we should make sure they get set
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'instance_type': 'm3.medium',
'keyname': 'bookey',
'group': 'boogroup',
'min_machines': 1,
'max_machines': 1,
'EC2_ACCESS_KEY': 'access key',
'EC2_SECRET_KEY': 'secret key',
'zone': 'my-zone-1b'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# finally, pretend that our ec2 zone/image to use exist
fake_ec2 = flexmock(name="fake_ec2")
fake_ec2.should_receive('get_all_instances')
fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \
.and_return('anything')
fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \
.and_return()
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').with_args(
'my-zone-1',
aws_access_key_id='access key',
aws_secret_access_key='secret key').and_return(fake_ec2)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
appscale.up()
self.assertEquals('access key', os.environ['EC2_ACCESS_KEY'])
self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def testUpWithCloudAppScalefile(self):
# calling 'appscale up' if there is an AppScalefile present
# should call appscale-run-instances with the given config
# params. here, we assume that the file is intended for use
# on EC2
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'instance_type': 'm3.medium',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'min_machines': 1,
'max_machines': 1,
'zone': 'my-zone-1b'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# throw in some mocks for the argument parsing
for credential in EC2Agent.REQUIRED_CREDENTIALS:
os.environ[credential] = "baz"
# finally, pretend that our ec2 zone and image exists
fake_ec2 = flexmock(name="fake_ec2")
fake_ec2.should_receive('get_all_instances')
fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \
.and_return('anything')
fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \
.and_return()
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').with_args(
'my-zone-1', aws_access_key_id='baz',
aws_secret_access_key='baz').and_return(fake_ec2)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
appscale.up()
def testUpWithCloudAppScalefile(self):
# calling 'appscale up' if there is an AppScalefile present
# should call appscale-run-instances with the given config
# params. here, we assume that the file is intended for use
# on EC2
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'instance_type' : 'm3.medium',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'min_machines' : 1,
'max_machines' : 1,
'zone' : 'my-zone-1b'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# throw in some mocks for the argument parsing
for credential in EC2Agent.REQUIRED_CREDENTIALS:
os.environ[credential] = "baz"
# finally, pretend that our ec2 zone and image exists
fake_ec2 = flexmock(name="fake_ec2")
fake_ec2.should_receive('get_all_instances')
fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \
.and_return('anything')
fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \
.and_return()
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').with_args('my-zone-1',
aws_access_key_id='baz', aws_secret_access_key='baz').and_return(fake_ec2)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
appscale.up()
def testUpWithEC2EnvironmentVariables(self):
# if the user wants us to use their EC2 credentials when running AppScale,
# we should make sure they get set
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'instance_type' : 'm3.medium',
'keyname' : 'bookey',
'group' : 'boogroup',
'min_machines' : 1,
'max_machines' : 1,
'EC2_ACCESS_KEY' : 'access key',
'EC2_SECRET_KEY' : 'secret key',
'zone' : 'my-zone-1b'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# finally, pretend that our ec2 zone/image to use exist
fake_ec2 = flexmock(name="fake_ec2")
fake_ec2.should_receive('get_all_instances')
fake_ec2.should_receive('get_all_zones').with_args('my-zone-1b') \
.and_return('anything')
fake_ec2.should_receive('get_image').with_args('ami-ABCDEFG') \
.and_return()
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').with_args('my-zone-1',
aws_access_key_id='access key',
aws_secret_access_key='secret key').and_return(fake_ec2)
# finally, mock out the actual appscale-run-instances call
flexmock(AppScaleTools)
AppScaleTools.should_receive('run_instances')
appscale.up()
self.assertEquals('access key', os.environ['EC2_ACCESS_KEY'])
self.assertEquals('secret key', os.environ['EC2_SECRET_KEY'])
def get(self, property_regex):
""" 'get' provides a cleaner experience for users than the
appscale-get-property command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
property_regex: A regular expression indicating which AppController
properties should be retrieved.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-get-property command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--property")
command.append(property_regex)
# and exec it
options = ParseArgs(command, "appscale-get-property").args
return AppScaleTools.get_property(options)
def get(self, property_regex):
""" 'get' provides a cleaner experience for users than the
appscale-get-property command, by using the configuration options present in
the AppScalefile found in the current working directory.
Args:
property_regex: A regular expression indicating which AppController
properties should be retrieved.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# construct the appscale-get-property command
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--property")
command.append(property_regex)
# and exec it
options = ParseArgs(command, "appscale-get-property").args
return AppScaleTools.get_property(options)
def deploy(self, app, project_id=None):
""" 'deploy' is a more accessible way to tell an AppScale deployment to run a
Google App Engine application than 'appscale-upload-app'. It calls that
command with the configuration options found in the AppScalefile in the
current working directory.
Args:
app: The path (absolute or relative) to the Google App Engine application
that should be uploaded.
project_id: Which project ID to use to deploy the application.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct an upload-app command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
command.append("--verbose")
command.append("--file")
command.append(app)
if project_id is not None:
command.append("--project")
command.append(project_id)
# Finally, exec the command. Don't worry about validating it -
# appscale-upload-app will do that for us.
options = ParseArgs(command, "appscale-upload-app").args
login_host, http_port = AppScaleTools.upload_app(options)
AppScaleTools.update_cron(options.file, options.keyname)
AppScaleTools.update_queues(options.file, options.keyname)
return login_host, http_port
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(AdminClient).should_receive('list_services').\
and_return(['default'])
flexmock(AdminClient).should_receive('delete_service').\
with_args('blargapp', 'default').and_return('op_id')
flexmock(AdminClient).should_receive('get_operation').\
with_args('blargapp', 'op_id').and_return({'done': True})
argv = ["--project-id", "blargapp", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow", "load_balancer"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(AdminClient).should_receive('list_services').\
and_return(['default'])
flexmock(AdminClient).should_receive('delete_service').\
with_args('blargapp', 'default').and_return('op_id')
flexmock(AdminClient).should_receive('get_operation').\
with_args('blargapp', 'op_id').and_return({'done': True})
argv = [
"--project-id", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def testUpWithClusterAppScalefile(self):
# calling 'appscale up' if there is an AppScalefile present
# should call appscale-run-instances with the given config
# params. here, we assume that the file is intended for use
# on a virtualized cluster
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout': {
'master': 'ip1',
'appengine': 'ip1',
'database': 'ip2',
'zookeeper': 'ip2'
},
'keyname': 'boobazblarg',
'group': 'boobazblarg'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# for this test, let's say that we don't have an SSH key already
# set up for ip1 and ip2
# TODO(cgb): Add in tests where we have a key for ip1 but not ip2,
# and the case where we have a key but it doesn't work
key_path = os.path.expanduser('~/.appscale/boobazblarg.key')
os.path.should_receive('exists').with_args(key_path).and_return(False)
# finally, mock out the actual appscale tools calls. since we're running
# via a cluster, this means we call add-keypair to set up SSH keys, then
# run-instances to start appscale
flexmock(AppScaleTools)
AppScaleTools.should_receive('add_keypair')
AppScaleTools.should_receive('run_instances')
appscale.up()
def status(self, extra_options_list=None):
""" 'status' is a more accessible way to query the state of the AppScale
deployment than 'appscale-describe-instances', and calls it with the
parameters in the user's AppScalefile.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
"""
contents = self.read_appscalefile()
# Construct a describe-instances command from the file's contents
command = extra_options_list or []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
# Finally, exec the command. Don't worry about validating it -
# appscale-describe-instances will do that for us.
options = ParseArgs(command, "appscale-describe-instances").args
AppScaleTools.print_cluster_status(options)
def testCreateUserWithAppScalefile(self):
# calling 'appscale create-user' with AppScalefile in the local
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min_machines': 1,
'max_machines': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-create-user call
flexmock(AppScaleTools)
AppScaleTools.should_receive('create_user')
appscale.create_user()
def status(self, extra_options_list=None):
""" 'status' is a more accessible way to query the state of the AppScale
deployment than 'appscale-describe-instances', and calls it with the
parameters in the user's AppScalefile.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
"""
contents = self.read_appscalefile()
# Construct a describe-instances command from the file's contents
command = extra_options_list or []
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
# Finally, exec the command. Don't worry about validating it -
# appscale-describe-instances will do that for us.
options = ParseArgs(command, "appscale-describe-instances").args
AppScaleTools.print_cluster_status(options)
def testCreateUserWithAppScalefile(self):
# calling 'appscale create-user' with AppScalefile in the local
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min_machines': 1,
'max_machines': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-create-user call
flexmock(AppScaleTools)
AppScaleTools.should_receive('create_user')
appscale.create_user()
def testUpWithClusterAppScalefile(self):
# calling 'appscale up' if there is an AppScalefile present
# should call appscale-run-instances with the given config
# params. here, we assume that the file is intended for use
# on a virtualized cluster
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout': {'master': 'ip1', 'appengine': 'ip1',
'database': 'ip2', 'zookeeper': 'ip2'},
'keyname': 'boobazblarg',
'group': 'boobazblarg'
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
'/boo/' + appscale.APPSCALEFILE).and_return(True)
# for this test, let's say that we don't have an SSH key already
# set up for ip1 and ip2
# TODO(cgb): Add in tests where we have a key for ip1 but not ip2,
# and the case where we have a key but it doesn't work
key_path = os.path.expanduser('~/.appscale/boobazblarg.key')
os.path.should_receive('exists').with_args(key_path).and_return(False)
# finally, mock out the actual appscale tools calls. since we're running
# via a cluster, this means we call add-keypair to set up SSH keys, then
# run-instances to start appscale
flexmock(AppScaleTools)
AppScaleTools.should_receive('add_keypair')
AppScaleTools.should_receive('run_instances')
appscale.up()
def test_get_property(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out finding the shadow node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'shadow']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
property_name = "name"
property_value = "value"
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('set_property').with_args(
property_name, property_value, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname, "--property_name", property_name,
"--property_value", property_value
]
options = ParseArgs(argv, self.function).args
result = AppScaleTools.set_property(options)
self.assertEqual(None, result)
def relocate(self, appid, http_port, https_port):
""" 'relocate' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
appid: A str indicating the name of the application to relocate.
http_port: An int that indicates what port should serve HTTP traffic for
this application.
https_port: An int that indicates what port should serve HTTPS traffic for
this application.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# Construct the appscale-relocate-app command from argv and the contents of
# the AppScalefile.
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--appname")
command.append(appid)
command.append("--http_port")
command.append(str(http_port))
command.append("--https_port")
command.append(str(https_port))
# and exec it
options = ParseArgs(command, "appscale-relocate-app").args
AppScaleTools.relocate_app(options)
def relocate(self, appid, http_port, https_port):
""" 'relocate' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
appid: A str indicating the name of the application to relocate.
http_port: An int that indicates what port should serve HTTP traffic for
this application.
https_port: An int that indicates what port should serve HTTPS traffic for
this application.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# Construct the appscale-relocate-app command from argv and the contents of
# the AppScalefile.
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
command.append("--appname")
command.append(appid)
command.append("--http_port")
command.append(str(http_port))
command.append("--https_port")
command.append(str(https_port))
# and exec it
options = ParseArgs(command, "appscale-relocate-app").args
AppScaleTools.relocate_app(options)
def testRelocateWithAppScalefile(self):
# calling 'appscale relocate' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-relocate-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'verbose' : True,
'min' : 1,
'max' : 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-relocate-app call
flexmock(AppScaleTools)
AppScaleTools.should_receive('relocate_app')
appscale.relocate('myapp', 80, 443)
def testSetPropertyWithAppScalefile(self):
# calling 'appscale set' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-get-property' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure' : 'ec2',
'machine' : 'ami-ABCDEFG',
'keyname' : 'bookey',
'group' : 'boogroup',
'verbose' : True,
'min_machines' : 1,
'max_machines' : 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-set-property call
flexmock(AppScaleTools)
AppScaleTools.should_receive('set_property')
appscale.set('key', 'value')
def testRelocateWithAppScalefile(self):
# calling 'appscale relocate' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-relocate-app' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min': 1,
'max': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-relocate-app call
flexmock(AppScaleTools)
AppScaleTools.should_receive('relocate_app')
appscale.relocate('myapp', 80, 443)
def testSetPropertyWithAppScalefile(self):
# calling 'appscale set' with an AppScalefile in the local
# directory should collect any parameters needed for the
# 'appscale-get-property' command and then exec it
appscale = AppScale()
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'infrastructure': 'ec2',
'machine': 'ami-ABCDEFG',
'keyname': 'bookey',
'group': 'boogroup',
'verbose': True,
'min_machines': 1,
'max_machines': 1
}
yaml_dumped_contents = yaml.dump(contents)
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
# finally, mock out the actual appscale-set-property call
flexmock(AppScaleTools)
AppScaleTools.should_receive('set_property')
appscale.set('key', 'value')
def create_user(self, is_admin=False):
""" 'create_user' Creates a new user from the tools.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
TypeError: If the user does not provide an new user credentials
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
command = []
if 'keyname' in contents_as_yaml:
command.append("--keyname")
command.append(contents_as_yaml["keyname"])
options = ParseArgs(command, "appscale-create-user").args
return AppScaleTools.create_user(options, is_admin)
