def test_remove_app_but_app_isnt_running(self):
# mock out reading from stdin, and assume the user says 'yes'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('yes')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
app_stats_data = {
'apps': {
'pippo': {
'http': 8080,
'language': 'python27',
'total_reqs': 'no_change',
'appservers': 1,
'https': 4380,
'reqs_enqueued': None
}
}
}
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
argv = ["--appname", "blargapp", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.remove_app, options)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
app_stats_data = {'apps': {'blargapp': {'http': 8080, 'language': 'python27',
'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('stop_app').with_args('blargapp',
'the secret').and_return('OK')
fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
'the secret').and_return(True).and_return(True).and_return(False)
fake_appcontroller.should_receive('does_app_exist').with_args('blargapp',
'the secret').and_return(True)
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(RemoteHelper).should_receive('is_port_open').and_return(False)
argv = [
"--appname", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [
{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["appengine"]
},
]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def test_describe_instances_with_two_nodes(self):
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at 1.2.3.4')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps(
{"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["appengine"]
},
]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# assume that there are two machines running in our deployment
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.describe_instances(options)
def test_terminate_when_not_running(self):
# let's say that appscale isn't running, so we should throw up and die
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(False)
argv = ["--keyname", self.keyname, "--test"]
options = ParseArgs(argv, self.function).args
self.assertRaises(BadConfigurationException,
AppScaleTools.terminate_instances, options)
def test_ensure_appscale_isnt_running_but_it_is(self):
# if there is a secret file and force isn't set, we should abort
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
flexmock(LocalState).should_receive('get_login_host').and_return('login_ip')
flexmock(LocalState).should_receive('get_secret_key').and_return('super-secret')
flexmock(AppControllerClient).should_receive('get_status').and_return("OK")
self.assertRaises(BadConfigurationException,
LocalState.ensure_appscale_isnt_running, self.keyname,
False)
def test_reset_password_for_user_that_exists(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'db_master']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('reset_password').with_args(
'*****@*****.**', str, 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.reset_password(options)
def test_reset_password_for_user_that_exists(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the username and new password from the user
builtins.should_receive('raw_input').and_return('*****@*****.**')
flexmock(getpass)
getpass.should_receive('getpass').and_return('the password')
# mock out finding the login node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'db_master']
}]}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('nothing interesting here') \
.and_return('Database is at not-up-yet') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('reset_password').with_args(
'*****@*****.**', str, 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.reset_password(options)
def test_get_property(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out finding the shadow node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['login', 'shadow']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
property_name = "name"
property_value = "value"
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('set_property').with_args(
property_name, property_value, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname, "--property_name", property_name,
"--property_value", property_value
]
options = ParseArgs(argv, self.function).args
result = AppScaleTools.set_property(options)
self.assertEqual(None, result)
def test_terminate_when_not_running(self):
# let's say that appscale isn't running, so we should throw up and die
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(False)
argv = [
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
self.assertRaises(BadConfigurationException,
AppScaleTools.terminate_instances,
options)
def test_ensure_appscale_isnt_running_but_it_is(self):
# if there is a secret file and force isn't set, we should abort
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
flexmock(LocalState).should_receive('get_login_host').and_return(
'login_ip')
flexmock(LocalState).should_receive('get_secret_key').and_return(
'super-secret')
(flexmock(AppControllerClient).should_receive(
'get_all_public_ips').and_return("OK"))
self.assertRaises(BadConfigurationException,
LocalState.ensure_appscale_isnt_running,
self.keyname, False)
def test_remove_app_but_app_isnt_running(self):
# mock out reading from stdin, and assume the user says 'yes'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('yes')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
'the secret').and_return(False)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
argv = [
"--appname", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.remove_app, options)
def test_update_local_metadata(self):
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1']))
role_info = [{
'public_ip': 'public1',
'private_ip': 'private1',
'jobs': ['shadow', 'db_master']
}]
fake_soap.should_receive('get_role_info').with_args('the secret') \
.and_return(json.dumps(role_info))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_soap)
# mock out reading the secret key
fake_secret = flexmock(name='fake_secret')
fake_secret.should_receive('read').and_return('the secret')
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location('booscale'), 'r') \
.and_return(fake_secret)
# Mock out writing the json file.
json_location = LocalState.get_locations_json_location('booscale')
builtins.should_receive('open').with_args(json_location, 'w')\
.and_return(flexmock(write=lambda *args: None))
options = flexmock(name='options',
table='cassandra',
infrastructure='ec2',
keyname='booscale',
group='boogroup',
zone='my-zone-1b',
EC2_ACCESS_KEY='baz',
EC2_SECRET_KEY='baz',
EC2_URL='')
node_layout = NodeLayout(
options={
'min_machines': 1,
'max_machines': 1,
'infrastructure': 'ec2',
'table': 'cassandra',
'instance_type': 'm1.large'
})
LocalState.update_local_metadata(options, 'public1', 'public1')
def test_get_property(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out finding the shadow node's IP address from the json file
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
'public_ip': 'public1',
'private_ip': 'private1',
'roles': ['shadow']
}]}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out grabbing the userappserver ip from an appcontroller
property_name = "name"
property_value = "value"
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('set_property').with_args(property_name,
property_value, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--keyname", self.keyname,
"--property_name", property_name,
"--property_value", property_value
]
options = ParseArgs(argv, self.function).args
result = AppScaleTools.set_property(options)
self.assertEqual(None, result)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow", "load_balancer"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(AdminClient).should_receive('list_services').\
and_return(['default'])
flexmock(AdminClient).should_receive('delete_service').\
with_args('blargapp', 'default').and_return('op_id')
flexmock(AdminClient).should_receive('get_operation').\
with_args('blargapp', 'op_id').and_return({'done': True})
argv = [
"--project-id", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def setUp(self):
self.keyname = "boobazblargfoo"
self.function = "appscale-relocate-app"
self.appid = 'my-crazy-app'
# mock out any writing to stdout
flexmock(AppScaleLogger)
AppScaleLogger.should_receive('log').and_return()
AppScaleLogger.should_receive('success').and_return()
AppScaleLogger.should_receive('warn').and_return()
# mock out all sleeping
flexmock(time)
time.should_receive('sleep').and_return()
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]
}))
fake_nodes_json.should_receive('write').and_return()
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# put in a mock for reading the secret file
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
def test_remove_app_and_app_is_running(self):
# mock out reading from stdin, and assume the user says 'YES'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('YES')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(AdminClient).should_receive('list_services').\
and_return(['default'])
flexmock(AdminClient).should_receive('delete_service').\
with_args('blargapp', 'default').and_return('op_id')
flexmock(AdminClient).should_receive('get_operation').\
with_args('blargapp', 'op_id').and_return({'done': True})
argv = ["--project-id", "blargapp", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.remove_app(options)
def setUp(self):
self.keyname = "boobazblargfoo"
self.function = "appscale-relocate-app"
self.appid = 'my-crazy-app'
# mock out any writing to stdout
flexmock(AppScaleLogger)
AppScaleLogger.should_receive('log').and_return()
AppScaleLogger.should_receive('success').and_return()
AppScaleLogger.should_receive('warn').and_return()
# mock out all sleeping
flexmock(time)
time.should_receive('sleep').and_return()
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# put in a mock for reading the secret file
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
def test_remove_app_but_app_isnt_running(self):
# mock out reading from stdin, and assume the user says 'yes'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('yes')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
flexmock(AdminClient).should_receive('delete_project').\
and_raise(AdminError)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
argv = [
"--project-id", "blargapp",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AdminError, AppScaleTools.remove_app, options)
def test_update_local_metadata(self):
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1']))
role_info = [{
'public_ip' : 'public1',
'private_ip' : 'private1',
'roles' : ['shadow', 'db_master']
}]
fake_soap.should_receive('get_role_info').with_args('the secret') \
.and_return(json.dumps(role_info))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_soap)
# mock out reading the secret key
fake_secret = flexmock(name='fake_secret')
fake_secret.should_receive('read').and_return('the secret')
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location('booscale'), 'r') \
.and_return(fake_secret)
# Mock out writing the json file.
json_location = LocalState.get_locations_json_location('booscale')
builtins.should_receive('open').with_args(json_location, 'w')\
.and_return(flexmock(write=lambda *args: None))
options = flexmock(name='options', table='cassandra', infrastructure='ec2',
keyname='booscale', group='boogroup', zone='my-zone-1b',
EC2_ACCESS_KEY='baz', EC2_SECRET_KEY='baz', EC2_URL='')
node_layout = NodeLayout(options={
'min_machines' : 1,
'max_machines' : 1,
'infrastructure' : 'ec2',
'table' : 'cassandra',
'instance_type': 'm1.large'
})
LocalState.update_local_metadata(options, 'public1', 'public1')
def test_get_property(self):
# put in a mock for reading the secret file
builtins = flexmock(sys.modules["__builtin__"])
builtins.should_call("open") # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive("read").and_return("the secret")
builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)
# mock out finding the shadow node's IP address from the json file
flexmock(os.path)
os.path.should_call("exists") # set the fall-through
os.path.should_receive("exists").with_args(LocalState.get_locations_json_location(self.keyname)).and_return(
True
)
fake_nodes_json = flexmock(name="fake_secret")
fake_nodes_json.should_receive("read").and_return(
json.dumps({"node_info": [{"public_ip": "public1", "private_ip": "private1", "jobs": ["login", "shadow"]}]})
)
builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return(
fake_nodes_json
)
# mock out grabbing the userappserver ip from an appcontroller
property_name = "name"
property_value = "value"
fake_appcontroller = flexmock(name="fake_appcontroller")
fake_appcontroller.should_receive("set_property").with_args(
property_name, property_value, "the secret"
).and_return("OK")
flexmock(SOAPpy)
SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_appcontroller)
argv = ["--keyname", self.keyname, "--property_name", property_name, "--property_value", property_value]
options = ParseArgs(argv, self.function).args
result = AppScaleTools.set_property(options)
self.assertEqual(None, result)
def test_remove_app_but_app_isnt_running(self):
# mock out reading from stdin, and assume the user says 'yes'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_receive('raw_input').and_return('yes')
# mock out reading the secret key
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow", "load_balancer"]
}]
}))
fake_nodes_json.should_receive('write').and_return()
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
flexmock(AdminClient).should_receive('list_services').\
and_raise(AdminError)
argv = ["--project-id", "blargapp", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
self.assertRaises(AdminError, AppScaleTools.remove_app, options)
def test_upload_app_when_app_exists_on_virt_cluster(self):
# we do let you upload an app if it's already running
# add in mocks so that there is an app.yaml with an appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {
'owner': '*****@*****.**',
'hosts': {
'192.168.1.1': {
'http': 8080,
'https': 4380
}
}
}
app_stats_data = {
'apps': {
'baz': {
'http': 8080,
'language': 'python27',
'total_reqs': 'no_change',
'appservers': 1,
'https': 4380,
'reqs_enqueued': None
}
}
}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args(
'baz', '/opt/appscale/apps/baz.tar.gz',
'the secret').and_return('OK')
fake_appcontroller.should_receive('update').with_args(
['baz'], 'the secret').and_return('OK')
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# and mock out tarring and copying the app
subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile(
'/tmp/appscale-app-baz.tar.gz'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
False, 5).and_return()
self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
.format(self.keyname)), False, 5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('controller-17443.cfg'))
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance', state='pending',
key_name=self.keyname, id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance', state='running',
key_name=self.keyname, id='i-ABCDEFG',
ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
.and_return(no_instances) \
.and_return(no_instances).and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--instance_type", "m3.medium",
"--machine", "ami-ABCDEFG",
"--use_spot_instances",
"--max_spot_price", "1.23",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_terminate_in_cloud_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is running, so we should
# terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on a virtualized cluster
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
# Assume persistent disks are used.
flexmock(LocalState).should_receive('are_disks_used').and_return(True)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(
json.dumps({
"infrastructure_info": {
'infrastructure': 'ec2',
'group': self.group,
},
"node_info": [{
'public_ip': 'public1',
'jobs': ['shadow']
}, {
'public_ip': 'public2',
'jobs': ['appengine']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# mock out talking to EC2
fake_ec2 = flexmock(name='fake_ec2')
# let's say that three instances are running, and that two of them are in
# our deployment
fake_one_running = flexmock(name='fake_one',
key_name=self.keyname,
state='running',
id='i-ONE',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_two_running = flexmock(name='fake_two',
key_name=self.keyname,
state='running',
id='i-TWO',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_three_running = flexmock(name='fake_three',
key_name='abcdefg',
state='running',
id='i-THREE',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_reservation_running = flexmock(
name='fake_reservation',
instances=[fake_one_running, fake_two_running, fake_three_running])
fake_one_terminated = flexmock(name='fake_one',
key_name=self.keyname,
state='terminated',
id='i-ONE',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_two_terminated = flexmock(name='fake_two',
key_name=self.keyname,
state='terminated',
id='i-TWO',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_three_terminated = flexmock(name='fake_three',
key_name='abcdefg',
state='terminated',
id='i-THREE',
ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_reservation_terminated = flexmock(name='fake_reservation',
instances=[
fake_one_terminated,
fake_two_terminated,
fake_three_terminated
])
fake_ec2.should_receive('get_all_instances').and_return(fake_reservation_running) \
.and_return(fake_reservation_terminated)
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').and_return(fake_ec2)
# and mock out the call to kill the instances
fake_ec2.should_receive('terminate_instances').with_args(
['i-ONE',
'i-TWO']).and_return([fake_one_terminated, fake_two_terminated])
# mock out the call to delete the keypair
fake_ec2.should_receive('delete_key_pair').and_return()
# and the call to delete the security group - let's say that we can't
# delete the group the first time, and can the second
fake_ec2.should_receive('delete_security_group').and_return(False) \
.and_return(True)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
flexmock(RemoteHelper).should_receive('terminate_cloud_infrastructure')
# also mock out asking the user for confirmation on shutting down
# their cloud
builtins.should_receive('raw_input').and_return('yes')
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".
format(options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException("Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".
format(options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.clean)
except (IOError, AppScaleException, AppControllerException,
BadConfigurationException) as e:
if not (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
raise
if options.test:
AppScaleLogger.warn(e)
else:
AppScaleLogger.verbose(e)
if isinstance(e, AppControllerException):
response = raw_input(
'AppScale may not have shut down properly, are you sure you want '
'to continue terminating? (y/N) ')
else:
response = raw_input(
'AppScale could not find the configuration files for this '
'deployment, are you sure you want to continue terminating? '
'(y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled cloud termination.")
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
RemoteHelper.terminate_cloud_infrastructure(options.keyname)
elif infrastructure in InfrastructureAgentFactory.VALID_AGENTS and not \
options.terminate:
AppScaleLogger.log("AppScale did not terminate any of your cloud "
"instances, to terminate them run 'appscale "
"down --terminate'")
if options.clean:
LocalState.clean_local_metadata(keyname=options.keyname)
def test_add_nodes_in_virt_deployment(self):
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_yaml = """
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
"""
# mock out reading the yaml file, and slip in our own
fake_yaml_file = flexmock(name='fake_yaml')
fake_yaml_file.should_receive('read').and_return(ips_yaml)
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
builtins.should_receive('open').with_args('/tmp/boo.yaml', 'r') \
.and_return(fake_yaml_file)
# and pretend we're on a virtualized cluster
locations_yaml = yaml.dump({
"infrastructure" : "xen"
})
# say that the ssh key works
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('ssh'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT,
stdin=self.fake_input_file) \
.and_return(self.success)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps({
"node_info": [{ "public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"] }]}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
json_node_info = json.dumps(yaml.safe_load(ips_yaml))
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('start_roles_on_nodes') \
.with_args(json_node_info, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = [
"--ips", "/tmp/boo.yaml",
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.add_instances(options)
def test_upload_php_app_successfully(self):
app_dir = '/tmp/appscale-app-1234'
# add in mocks so that the gzip'ed file gets extracted to /tmp
# as well as for removing it later
flexmock(os)
os.should_receive('mkdir').with_args(app_dir) \
.and_return(True)
flexmock(shutil)
shutil.should_receive('rmtree').with_args(app_dir).and_return()
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('tar zxvf'),False)\
.and_return()
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'php'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {'owner' : '*****@*****.**',
'hosts' : {'192.168.1.1' : { 'http' : 8080, 'https' : 4380 }}}
app_stats_data = {'apps': {'baz': {'http': 8080, 'language': 'python27',
'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args('baz',
'/opt/appscale/apps/baz.tar.gz', 'the secret').and_return()
fake_appcontroller.should_receive('update').with_args(['baz'],
'the secret').and_return()
fake_appcontroller.should_receive('is_app_running').with_args('baz',
'the secret').and_return(False).and_return(True)
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
local_state.should_receive('shell') \
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('^mkdir -p')) \
.and_return()
# and mock out tarring and copying the app
local_state.should_receive('shell') \
.with_args(re.compile('tar -czf'), False) \
.and_return()
local_state.should_receive('shell') \
.with_args(re.compile('/tmp/appscale-app-baz.tar.gz'), False, 5) \
.and_return()
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
os.should_receive('listdir').and_return(['app.yaml','index.py'])
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir + ".tar.gz"
]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def test_terminate_in_gce_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is running, so we should
# terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_client_secrets_location(
self.keyname)).and_return(True)
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on GCE
project_id = "1234567890"
zone = 'my-zone-1b'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
# Assume persistent disks are used.
flexmock(LocalState).should_receive('are_disks_used').and_return(True)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(
json.dumps({
"infrastructure_info": {
'infrastructure': 'gce',
'group': self.group,
'project': project_id,
'zone': zone
},
"node_info": [{
'public_ip': 'public1',
'jobs': ['shadow']
}, {
'public_ip': 'public2',
'jobs': ['appengine']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# also add in a fake client-secrets file for GCE
client_secrets = LocalState.get_client_secrets_location(self.keyname)
# mock out talking to GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive(
'flow_from_clientsecrets').with_args(
client_secrets, scope=str).and_return(fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
fake_flags = oauth2client.tools.argparser.parse_args(args=[])
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(str).and_return(
fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run_flow').with_args(
fake_flow, fake_storage, fake_flags).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_gce = flexmock(name='fake_gce')
# let's say that two instances are running
instance_one_info = {
u'status':
u'RUNNING',
u'kind':
u'compute#instance',
u'machineType':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name':
u'bazboogroup-one',
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {
u'fingerprint': u'42WmSpB8rSM='
},
u'image':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward':
False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email':
u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp':
u'2013-05-22T11:52:33.254-07:00',
u'id':
u'8684033495853907982',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP':
u'private1',
u'network':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name':
u'nic0'
}]
}
instance_two_info = {
u'status':
u'RUNNING',
u'kind':
u'compute#instance',
u'machineType':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name':
u'bazboogroup-two',
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {
u'fingerprint': u'42WmSpB8rSM='
},
u'image':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward':
False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email':
u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp':
u'2013-05-22T11:52:33.254-07:00',
u'id':
u'8684033495853907982',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP':
u'private1',
u'network':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name':
u'nic0'
}]
}
list_instance_info = {
u'items': [instance_one_info, instance_two_info],
u'kind':
u'compute#instanceList',
u'id':
u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(
name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_instances.should_receive('list').with_args(project=project_id,
filter="name eq bazboogroup-.*", zone=zone) \
.and_return(fake_list_instance_request)
fake_gce.should_receive('instances').and_return(fake_instances)
# And assume that we can kill both of our instances fine
delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de'
delete_instance_info_one = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
delete_instance_info_two = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
fake_delete_instance_request_one = flexmock(
name='fake_delete_instance_request_one')
fake_delete_instance_request_one.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_one)
fake_instances.should_receive('delete').with_args(
project=project_id, zone=zone,
instance='bazboogroup-one').and_return(
fake_delete_instance_request_one)
fake_delete_instance_request_two = flexmock(
name='fake_delete_instance_request_two')
fake_delete_instance_request_two.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_two)
fake_instances.should_receive('delete').with_args(
project=project_id, zone=zone,
instance='bazboogroup-two').and_return(
fake_delete_instance_request_two)
# mock out our waiting for the instances to be deleted
all_done = {u'status': u'DONE'}
fake_instance_checker = flexmock(name='fake_instance_checker')
fake_instance_checker.should_receive('execute').and_return(all_done)
fake_blocker = flexmock(name='fake_blocker')
fake_blocker.should_receive('get').with_args(
project=project_id, operation=delete_instance,
zone=zone).and_return(fake_instance_checker)
fake_gce.should_receive('zoneOperations').and_return(fake_blocker)
# mock out the call to delete the firewall
delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654'
fake_delete_firewall_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_firewall,
u'startTime': u'2013-05-27T11:01:35.482-07:00',
u'insertTime': u'2013-05-27T11:01:35.390-07:00',
u'targetId': u'11748720697396371259',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'15574488986772298961',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654',
u'user': u'*****@*****.**'
}
fake_delete_firewall_request = flexmock(
name='fake_delete_firewall_request')
fake_delete_firewall_request.should_receive('execute').and_return(
fake_delete_firewall_info)
fake_firewalls = flexmock(name='fake_firewalls')
fake_firewalls.should_receive('delete').with_args(
project=project_id,
firewall=self.group).and_return(fake_delete_firewall_request)
fake_gce.should_receive('firewalls').and_return(fake_firewalls)
# mock out the call to make sure the firewall was deleted
fake_firewall_checker = flexmock(name='fake_firewall_checker')
fake_firewall_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(
project=project_id,
operation=delete_firewall).and_return(fake_firewall_checker)
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
# and the call to delete the network
delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca'
fake_delete_network_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_network,
u'startTime': u'2013-05-27T11:02:30.012-07:00',
u'insertTime': u'2013-05-27T11:02:29.954-07:00',
u'targetId': u'17688075350400527692',
u'targetLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'12623697331874594836',
u'selfLink':
u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca',
u'user': u'*****@*****.**'
}
fake_delete_network_request = flexmock(
name='fake_delete_network_request')
fake_delete_network_request.should_receive('execute').and_return(
fake_delete_network_info)
fake_networks = flexmock(name='fake_networks')
fake_networks.should_receive('delete').with_args(
project=project_id,
network=self.group).and_return(fake_delete_network_request)
fake_gce.should_receive('networks').and_return(fake_networks)
# mock out the call to make sure the network was deleted
fake_network_checker = flexmock(name='fake_network_checker')
fake_network_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(
project=project_id,
operation=delete_network).and_return(fake_network_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args(
'compute', GCEAgent.API_VERSION).and_return(fake_gce)
flexmock(GCEAgent).should_receive('get_secrets_type')\
.and_return(CredentialTypes.OAUTH)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
flexmock(RemoteHelper).should_receive('terminate_cloud_infrastructure')
argv = ["--keyname", self.keyname, "--test"]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()
# next, mock out finding the login ip address
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]
}))
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args(
'the secret').and_return(json.dumps(['public1', 'public2']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# fake the creation of the log directories locally
os.should_receive('mkdir').with_args(
'/tmp/foobaz/public1').and_return()
os.should_receive('mkdir').with_args('/tmp/foobaz/public1/cassandra')
os.should_receive('mkdir').with_args(
'/tmp/foobaz/public2').and_return()
os.should_receive('mkdir').with_args('/tmp/foobaz/public2/cassandra')
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/opt/cassandra/cassandra/logs'),
shell=True,
stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def test_terminate_in_gce_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is running, so we should
# terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_client_secrets_location(self.keyname)).and_return(True)
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on GCE
project_id = "1234567890"
zone = 'my-zone-1b'
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
# Assume persistent disks are used.
flexmock(LocalState).should_receive('are_disks_used').and_return(True)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(json.dumps({
"infrastructure_info": {
'infrastructure': 'gce',
'group': self.group,
'project': project_id,
'zone': zone
},
"node_info": [
{
'public_ip': 'public1',
'jobs': ['shadow']
},
{
'public_ip': 'public2',
'jobs': ['appengine']
}
]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# also add in a fake client-secrets file for GCE
client_secrets = LocalState.get_client_secrets_location(self.keyname)
# mock out talking to GCE
# first, mock out the oauth library calls
fake_flow = flexmock(name='fake_flow')
flexmock(oauth2client.client)
oauth2client.client.should_receive('flow_from_clientsecrets').with_args(
client_secrets, scope=str).and_return(fake_flow)
fake_storage = flexmock(name='fake_storage')
fake_storage.should_receive('get').and_return(None)
fake_flags = oauth2client.tools.argparser.parse_args(args=[])
flexmock(oauth2client.file)
oauth2client.file.should_receive('Storage').with_args(str).and_return(
fake_storage)
fake_credentials = flexmock(name='fake_credentials')
flexmock(oauth2client.tools)
oauth2client.tools.should_receive('run_flow').with_args(fake_flow,
fake_storage, fake_flags).and_return(fake_credentials)
# next, mock out http calls to GCE
fake_http = flexmock(name='fake_http')
fake_authorized_http = flexmock(name='fake_authorized_http')
flexmock(httplib2)
httplib2.should_receive('Http').and_return(fake_http)
fake_credentials.should_receive('authorize').with_args(fake_http) \
.and_return(fake_authorized_http)
fake_gce = flexmock(name='fake_gce')
# let's say that two instances are running
instance_one_info = {
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': u'bazboogroup-one',
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP': u'private1',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name': u'nic0'
}]
}
instance_two_info = {
u'status': u'RUNNING',
u'kind': u'compute#instance',
u'machineType': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/machineTypes/n1-standard-1',
u'name': u'bazboogroup-two',
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'tags': {u'fingerprint': u'42WmSpB8rSM='},
u'image': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/images/lucid64',
u'disks': [{
u'index': 0,
u'kind': u'compute#attachedDisk',
u'type': u'EPHEMERAL',
u'mode': u'READ_WRITE'
}],
u'canIpForward': False,
u'serviceAccounts': [{
u'scopes': [GCEAgent.GCE_SCOPE],
u'email': u'*****@*****.**'
}],
u'metadata': {
u'kind': u'compute#metadata',
u'fingerprint': u'42WmSpB8rSM='
},
u'creationTimestamp': u'2013-05-22T11:52:33.254-07:00',
u'id': u'8684033495853907982',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/bazboogroup-feb10b11-62bc-4536-ac25-9734f2267d6d',
u'networkInterfaces': [{
u'accessConfigs': [{
u'kind': u'compute#accessConfig',
u'type': u'ONE_TO_ONE_NAT',
u'name': u'External NAT',
u'natIP': u'public1'
}],
u'networkIP': u'private1',
u'network': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/bazboogroup',
u'name': u'nic0'
}]
}
list_instance_info = {
u'items': [instance_one_info, instance_two_info],
u'kind': u'compute#instanceList',
u'id': u'projects/appscale.com:appscale/zones/my-zone-1b/instances',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/961228229472/zones/my-zone-1b/instances'
}
fake_list_instance_request = flexmock(name='fake_list_instance_request')
fake_list_instance_request.should_receive('execute').with_args(
http=fake_authorized_http).and_return(list_instance_info)
fake_instances = flexmock(name='fake_instances')
fake_instances.should_receive('list').with_args(project=project_id,
filter="name eq bazboogroup-.*", zone=zone) \
.and_return(fake_list_instance_request)
fake_gce.should_receive('instances').and_return(fake_instances)
# And assume that we can kill both of our instances fine
delete_instance = u'operation-1369676691806-4ddb6b4ab6f39-a095d3de'
delete_instance_info_one = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
delete_instance_info_two = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_instance,
u'zone': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b',
u'startTime': u'2013-05-27T10:44:51.849-07:00',
u'insertTime': u'2013-05-27T10:44:51.806-07:00',
u'targetId': u'12912855597472179535',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/instances/appscalecgb20-0cf89267-5887-4048-b774-ca20de47a07f',
u'operationType': u'delete',
u'progress': 0,
u'id': u'11114355109942058217',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/zones/my-zone-1b/operations/operation-1369676691806-4ddb6b4ab6f39-a095d3de',
u'user': u'*****@*****.**'
}
fake_delete_instance_request_one = flexmock(name='fake_delete_instance_request_one')
fake_delete_instance_request_one.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_one)
fake_instances.should_receive('delete').with_args(project=project_id,
zone=zone, instance='bazboogroup-one').and_return(
fake_delete_instance_request_one)
fake_delete_instance_request_two = flexmock(name='fake_delete_instance_request_two')
fake_delete_instance_request_two.should_receive('execute').with_args(
http=fake_authorized_http).and_return(delete_instance_info_two)
fake_instances.should_receive('delete').with_args(project=project_id,
zone=zone, instance='bazboogroup-two').and_return(
fake_delete_instance_request_two)
# mock out our waiting for the instances to be deleted
all_done = {
u'status' : u'DONE'
}
fake_instance_checker = flexmock(name='fake_instance_checker')
fake_instance_checker.should_receive('execute').and_return(all_done)
fake_blocker = flexmock(name='fake_blocker')
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_instance, zone=zone).and_return(
fake_instance_checker)
fake_gce.should_receive('zoneOperations').and_return(fake_blocker)
# mock out the call to delete the firewall
delete_firewall = u'operation-1369677695390-4ddb6f07cc611-5a8f1654'
fake_delete_firewall_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_firewall,
u'startTime': u'2013-05-27T11:01:35.482-07:00',
u'insertTime': u'2013-05-27T11:01:35.390-07:00',
u'targetId': u'11748720697396371259',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/firewalls/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'15574488986772298961',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677695390-4ddb6f07cc611-5a8f1654',
u'user': u'*****@*****.**'
}
fake_delete_firewall_request = flexmock(name='fake_delete_firewall_request')
fake_delete_firewall_request.should_receive('execute').and_return(fake_delete_firewall_info)
fake_firewalls = flexmock(name='fake_firewalls')
fake_firewalls.should_receive('delete').with_args(project=project_id,
firewall=self.group).and_return(fake_delete_firewall_request)
fake_gce.should_receive('firewalls').and_return(fake_firewalls)
# mock out the call to make sure the firewall was deleted
fake_firewall_checker = flexmock(name='fake_firewall_checker')
fake_firewall_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_firewall).and_return(fake_firewall_checker)
fake_gce.should_receive('globalOperations').and_return(fake_blocker)
# and the call to delete the network
delete_network = u'operation-1369677749954-4ddb6f3bd1849-056cf8ca'
fake_delete_network_info = {
u'status': u'PENDING',
u'kind': u'compute#operation',
u'name': delete_network,
u'startTime': u'2013-05-27T11:02:30.012-07:00',
u'insertTime': u'2013-05-27T11:02:29.954-07:00',
u'targetId': u'17688075350400527692',
u'targetLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/networks/appscalecgb20',
u'operationType': u'delete',
u'progress': 0,
u'id': u'12623697331874594836',
u'selfLink': u'https://www.googleapis.com/compute/v1beta14/projects/appscale.com:appscale/global/operations/operation-1369677749954-4ddb6f3bd1849-056cf8ca',
u'user': u'*****@*****.**'
}
fake_delete_network_request = flexmock(name='fake_delete_network_request')
fake_delete_network_request.should_receive('execute').and_return(fake_delete_network_info)
fake_networks = flexmock(name='fake_networks')
fake_networks.should_receive('delete').with_args(project=project_id,
network=self.group).and_return(fake_delete_network_request)
fake_gce.should_receive('networks').and_return(fake_networks)
# mock out the call to make sure the network was deleted
fake_network_checker = flexmock(name='fake_network_checker')
fake_network_checker.should_receive('execute').and_return(all_done)
fake_blocker.should_receive('get').with_args(project=project_id,
operation=delete_network).and_return(fake_network_checker)
# finally, inject our fake GCE connection
flexmock(apiclient.discovery)
apiclient.discovery.should_receive('build').with_args('compute',
GCEAgent.API_VERSION).and_return(fake_gce)
flexmock(GCEAgent).should_receive('get_secrets_type')\
.and_return(CredentialTypes.OAUTH)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
flexmock(RemoteHelper).should_receive('terminate_cloud_infrastructure')
argv = [
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_terminate_in_virtual_cluster_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is
# running, so we should terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on a virtualized cluster
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
fake_yaml_file = flexmock(name='fake_file')
fake_yaml_file.should_receive('read').and_return(yaml.dump({
'infrastructure' : 'xen'
}))
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(json.dumps({"node_info": [
{
'public_ip': 'public1',
'jobs': ['shadow']
},
{
'public_ip': 'public2',
'jobs': ['appengine']
}
]}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# mock out talking to the appcontroller, and assume that it tells us there
# there are still two machines in this deployment
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# and mock out the ssh call to kill the remote appcontroller, assuming that
# it fails the first time and passes the second
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('controller stop'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.failed).and_return(self.success)
# next, mock out our checks to see how the stopping process is going and
# assume that it has stopped
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('ps x'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
# also mock out asking the user for confirmation on shutting down
# their cloud
builtins.should_receive('raw_input').and_return('yes')
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_terminate_in_virtual_cluster_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is
# running, so we should terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on a virtualized cluster
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
fake_yaml_file = flexmock(name='fake_file')
fake_yaml_file.should_receive('read').and_return(
yaml.dump({'infrastructure': 'xen'}))
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(
json.dumps({
"node_info": [{
'public_ip': 'public1',
'jobs': ['shadow']
}, {
'public_ip': 'public2',
'jobs': ['appengine']
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# mock out talking to the appcontroller, and assume that it tells us there
# there are still two machines in this deployment
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# and mock out the ssh call to kill the remote appcontroller, assuming that
# it fails the first time and passes the second
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('controller stop'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.failed).and_return(self.success)
# next, mock out our checks to see how the stopping process is going and
# assume that it has stopped
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('ps x'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
# also mock out asking the user for confirmation on shutting down
# their cloud
builtins.should_receive('raw_input').and_return('yes')
argv = ["--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_add_nodes_in_virt_deployment(self):
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_yaml = """
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
"""
# mock out reading the yaml file, and slip in our own
fake_yaml_file = flexmock(name='fake_yaml')
fake_yaml_file.should_receive('read').and_return(ips_yaml)
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
builtins.should_receive('open').with_args('/tmp/boo.yaml', 'r') \
.and_return(fake_yaml_file)
# and pretend we're on a virtualized cluster
locations_yaml = yaml.dump({"infrastructure": "xen"})
# say that the ssh key works
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('ssh'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT,
stdin=self.fake_input_file) \
.and_return(self.success)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps({
"node_info": [{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow", "load_balancer"]
}]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out the SOAP call to the AppController and assume it succeeded
json_node_info = json.dumps(yaml.safe_load(ips_yaml))
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('start_roles_on_nodes') \
.with_args(json_node_info, 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
argv = ["--ips", "/tmp/boo.yaml", "--keyname", self.keyname]
options = ParseArgs(argv, self.function).args
AppScaleTools.add_instances(options)
def test_terminate_in_cloud_and_succeeds(self):
# let's say that there is a locations.json file with key
# 'infrastructure_info', which means appscale is running, so we should
# terminate the services on each box
flexmock(os.path)
os.path.should_call('exists') # set up the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return(True)
# mock out reading the locations.json file with key
# 'infrastructure_info', and pretend that we're on a virtualized cluster
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
# Assume persistent disks are used.
flexmock(LocalState).should_receive('are_disks_used').and_return(True)
# mock out reading the json file, and pretend that we're running in a
# two node deployment
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_json_file = flexmock(name='fake_file')
fake_json_file.should_receive('read').and_return(json.dumps({
"infrastructure_info": {
'infrastructure': 'ec2',
'group': self.group,
},
"node_info": [
{
'public_ip': 'public1',
'jobs': ['shadow']
},
{
'public_ip': 'public2',
'jobs': ['appengine']
}
]
}))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_json_file)
# and slip in a fake secret file
fake_secret_file = flexmock(name='fake_file')
fake_secret_file.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(
LocalState.get_secret_key_location(self.keyname), 'r') \
.and_return(fake_secret_file)
# mock out talking to EC2
fake_ec2 = flexmock(name='fake_ec2')
# let's say that three instances are running, and that two of them are in
# our deployment
fake_one_running = flexmock(name='fake_one', key_name=self.keyname, state='running',
id='i-ONE', ip_address='1.2.3.4', private_ip_address='1.2.3.4')
fake_two_running = flexmock(name='fake_two', key_name=self.keyname, state='running',
id='i-TWO', ip_address='1.2.3.4', private_ip_address='1.2.3.4')
fake_three_running = flexmock(name='fake_three', key_name='abcdefg',
state='running', id='i-THREE', ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_reservation_running = flexmock(name='fake_reservation', instances=[fake_one_running,
fake_two_running, fake_three_running])
fake_one_terminated = flexmock(name='fake_one', key_name=self.keyname, state='terminated',
id='i-ONE', ip_address='1.2.3.4', private_ip_address='1.2.3.4')
fake_two_terminated = flexmock(name='fake_two', key_name=self.keyname, state='terminated',
id='i-TWO', ip_address='1.2.3.4', private_ip_address='1.2.3.4')
fake_three_terminated = flexmock(name='fake_three', key_name='abcdefg',
state='terminated', id='i-THREE', ip_address='1.2.3.4',
private_ip_address='1.2.3.4')
fake_reservation_terminated = flexmock(name='fake_reservation', instances=[fake_one_terminated,
fake_two_terminated, fake_three_terminated])
fake_ec2.should_receive('get_all_instances').and_return(fake_reservation_running) \
.and_return(fake_reservation_terminated)
flexmock(boto.ec2)
boto.ec2.should_receive('connect_to_region').and_return(fake_ec2)
# and mock out the call to kill the instances
fake_ec2.should_receive('terminate_instances').with_args(['i-ONE',
'i-TWO']).and_return([fake_one_terminated, fake_two_terminated])
# mock out the call to delete the keypair
fake_ec2.should_receive('delete_key_pair').and_return()
# and the call to delete the security group - let's say that we can't
# delete the group the first time, and can the second
fake_ec2.should_receive('delete_security_group').and_return(False) \
.and_return(True)
# finally, mock out removing the yaml file, json file, and secret key from
# this machine
flexmock(os)
os.should_receive('remove').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return()
os.should_receive('remove').with_args(
LocalState.get_secret_key_location(self.keyname)).and_return()
flexmock(RemoteHelper).should_receive('terminate_cloud_infrastructure')
# also mock out asking the user for confirmation on shutting down
# their cloud
builtins.should_receive('raw_input').and_return('yes')
argv = [
"--keyname", self.keyname
]
options = ParseArgs(argv, self.function).args
AppScaleTools.terminate_instances(options)
def test_upload_app_when_app_admin_not_this_user(self):
# we don't let you upload an app if the appid is registered to someone else,
# so abort
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {'owner': '*****@*****.**', 'hosts': {}}
app_data_not_admin = {'owner': '*****@*****.**'}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data_not_admin))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call("exists") # set the fall-through
os.path.should_receive("exists").with_args("/tmp/foobaz").and_return(False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive("mkdir").with_args("/tmp/foobaz").and_return()
# next, mock out finding the login ip address
os.path.should_receive("exists").with_args(LocalState.get_locations_json_location(self.keyname)).and_return(
True
)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive("read").and_return(
json.dumps({"node_info": [{"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"]}]})
)
builtins = flexmock(sys.modules["__builtin__"])
builtins.should_call("open")
builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return(
fake_nodes_json
)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive("read").and_return("the secret")
builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name="fake_appcontroller")
fake_appcontroller.should_receive("get_all_public_ips").with_args("the secret").and_return(
json.dumps(["public1", "public2"])
)
flexmock(SOAPpy)
SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_appcontroller)
# fake the creation of the log directories locally
os.should_receive("mkdir").with_args("/tmp/foobaz/public1").and_return()
os.should_receive("mkdir").with_args("/tmp/foobaz/public1/cassandra")
os.should_receive("mkdir").with_args("/tmp/foobaz/public2").and_return()
os.should_receive("mkdir").with_args("/tmp/foobaz/public2/cassandra")
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/appscale"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/kern.log*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/monit*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/nginx"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/syslog*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/var/log/zookeeper"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
subprocess.should_receive("Popen").with_args(
re.compile("/opt/cassandra/cassandra/logs"),
shell=True,
stdout=self.fake_temp_file,
stderr=subprocess.STDOUT,
).and_return(self.success)
argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def test_upload_php_app_successfully(self):
app_dir = '/tmp/appscale-app-1234'
# add in mocks so that the gzip'ed file gets extracted to /tmp
# as well as for removing it later
flexmock(os)
os.should_receive('mkdir').with_args(app_dir) \
.and_return(True)
flexmock(shutil)
shutil.should_receive('rmtree').with_args(app_dir).and_return()
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('tar zxvf'),False)\
.and_return()
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'php'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {
'owner': '*****@*****.**',
'hosts': {
'192.168.1.1': {
'http': 8080,
'https': 4380
}
}
}
app_stats_data = {
'apps': {
'baz': {
'http': 8080,
'language': 'python27',
'total_reqs': 'no_change',
'appservers': 1,
'https': 4380,
'reqs_enqueued': None
}
}
}
remote_tarball = '/opt/appscale/apps/baz.tar.gz'
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args(
'baz', remote_tarball, 'the secret').and_return()
fake_appcontroller.should_receive('update').with_args(
['baz'], 'the secret').and_return()
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
local_state.should_receive('shell') \
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('^mkdir -p')) \
.and_return()
# and mock out tarring and copying the app
local_state.should_receive('shell') \
.with_args(re.compile('tar -czf'), False) \
.and_return()
local_state.should_receive('shell') \
.with_args(re.compile('/tmp/appscale-app-baz.tar.gz'), False, 5) \
.and_return()
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
os.should_receive('listdir').and_return(['app.yaml', 'index.py'])
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
flexmock(RemoteHelper).should_receive('copy_app_to_host').\
and_return(remote_tarball)
argv = ["--keyname", self.keyname, "--file", self.app_dir + ".tar.gz"]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def test_upload_app_when_app_admin_not_this_user(self):
# we don't let you upload an app if the appid is registered to someone else,
# so abort
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {'owner' : '*****@*****.**','hosts' : { }}
app_data_not_admin = {'owner': '*****@*****.**'}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data_not_admin))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def test_appscale_in_one_node_virt_deployment(self):
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null [email protected] ",
False, 5, stdin="chmod +x /etc/init.d/appcontroller")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
rh = flexmock(RemoteHelper)
rh.should_receive('copy_deployment_credentials').and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').\
with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').\
with_args('logs.appscale.com').and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').\
with_args(secret_key_location, 'r').and_return(fake_secret)
self.builtins.should_receive('open').\
with_args(secret_key_location, 'w').and_return(fake_secret)
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# mock out copying over the keys
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.key'),False,5)
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell')\
.with_args(re.compile('^openssl'),False,stdin=None)\
.and_return()
# mock out removing the old json file
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume that we started monit fine
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
.and_return()
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null [email protected] ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.setup_socket_mocks('1.2.3.4')
self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip": "1.2.3.4",
"private_ip": "1.2.3.4",
"jobs": ["shadow", "login"]
}])))
# Assume the locations files were copied successfully.
locations_file = '{}/locations-bookey.yaml'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
.and_return()
locations_json = '{}/locations-bookey.json'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
.and_return()
user_locations = '/root/.appscale/locations-bookey.json'
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
.and_return()
# Assume the secret key was copied successfully.
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.secret'), False, 5)\
.and_return()
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(
False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()
# next, mock out finding the login ip address
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(
self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
nodes_info = {
"node_info": [
{
"public_ip":
"public1",
"private_ip":
"private1",
"jobs": [
"load_balancer", "taskqueue_master", "zookeeper",
"db_master", "taskqueue", "shadow", "login"
]
},
{
"public_ip": "public2",
"private_ip": "private2",
"jobs": ["memcache", "appengine", "zookeeper"]
},
{
"public_ip": "public3",
"private_ip": "private3",
"jobs": ["memcache", "appengine"]
},
]
}
fake_nodes_json.should_receive('read').and_return(
json.dumps(nodes_info))
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args(
'the secret').and_return(
json.dumps(['public1', 'public2', 'public3']))
fake_appcontroller.should_receive('get_role_info').with_args(
'the secret').and_return(json.dumps(nodes_info['node_info']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# fake the creation of the log directories locally
flexmock(utils)
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/private-ips')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public1')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/public1/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public2')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/public2/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public3')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/public3/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/load_balancer')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/taskqueue_master')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/zookeeper')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/db_master')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/taskqueue')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/login')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/memcache')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/appengine')
# fake creation of symlink to for friendly navigation
links_mapping = {
'../../public1': [
'/tmp/foobaz/symlinks/private-ips/private1',
'/tmp/foobaz/symlinks/load_balancer/public1',
'/tmp/foobaz/symlinks/taskqueue_master/public1',
'/tmp/foobaz/symlinks/zookeeper/public1',
'/tmp/foobaz/symlinks/db_master/public1',
'/tmp/foobaz/symlinks/taskqueue/public1',
'/tmp/foobaz/symlinks/shadow/public1',
'/tmp/foobaz/symlinks/login/public1',
],
'../../public2': [
'/tmp/foobaz/symlinks/private-ips/private2',
'/tmp/foobaz/symlinks/zookeeper/public2',
'/tmp/foobaz/symlinks/appengine/public2',
'/tmp/foobaz/symlinks/memcache/public2',
],
'../../public3': [
'/tmp/foobaz/symlinks/private-ips/private3',
'/tmp/foobaz/symlinks/appengine/public3',
'/tmp/foobaz/symlinks/memcache/public3',
]
}
for original_dir, expected_links in links_mapping.iteritems():
for expected_link in expected_links:
os.should_receive('symlink').with_args(original_dir,
expected_link)
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/var/log/haproxy*'),
shell=True,
stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/var/log/rabbitmq'),
shell=True,
stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/opt/cassandra/cassandra/logs'),
shell=True,
stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.23',
'ami-ABCDEFG', key_name=self.keyname, security_groups=['bazgroup'],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(re.compile('scp .*[r|d]sa'),
False, 5).and_return()
self.local_state.should_receive('shell').with_args(re.compile('scp .*{0}'
.format(self.keyname)), False, 5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('controller-17443.cfg'))
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazbargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/')
self.local_state.should_receive('shell').with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ', False, 5, stdin='chmod +x /etc/init.d/appcontroller').and_return()
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--instance_type", "m3.medium",
"--machine", "ami-ABCDEFG",
"--use_spot_instances",
"--max_spot_price", "1.23",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_two_node_virt_deployment(self):
# pretend that the place we're going to put logs into doesn't exist
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False)
# and mock out the mkdir operation
flexmock(os)
os.should_receive('mkdir').with_args('/tmp/foobaz').and_return()
# next, mock out finding the login ip address
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location(self.keyname)).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
nodes_info = {
"node_info": [
{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["load_balancer", "taskqueue_master", "zookeeper",
"db_master", "taskqueue", "shadow"]
}, {
"public_ip": "public2",
"private_ip": "private2",
"roles": ["memcache", "appengine", "zookeeper"]
}, {
"public_ip": "public3",
"private_ip": "private3",
"roles": ["memcache", "appengine"]
},
]
}
fake_nodes_json.should_receive('read').and_return(json.dumps(nodes_info))
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open')
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location(self.keyname), 'r') \
.and_return(fake_nodes_json)
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# and slip in a fake appcontroller to report on the two IP addrs
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('get_all_public_ips').with_args(
'the secret').and_return(json.dumps(['public1', 'public2', 'public3']))
fake_appcontroller.should_receive('get_role_info').with_args(
'the secret').and_return(json.dumps(nodes_info['node_info']))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_appcontroller)
# fake the creation of the log directories locally
flexmock(utils)
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/private-ips')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public1')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public2')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public3')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/cassandra')
utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/load_balancer')
utils.should_receive('mkdir').with_args(
'/tmp/foobaz/symlinks/taskqueue_master')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/zookeeper')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/db_master')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/taskqueue')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/memcache')
utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/appengine')
# fake creation of symlink to for friendly navigation
links_mapping = {
'../../public1': [
'/tmp/foobaz/symlinks/private-ips/private1',
'/tmp/foobaz/symlinks/load_balancer/public1',
'/tmp/foobaz/symlinks/taskqueue_master/public1',
'/tmp/foobaz/symlinks/zookeeper/public1',
'/tmp/foobaz/symlinks/db_master/public1',
'/tmp/foobaz/symlinks/taskqueue/public1',
'/tmp/foobaz/symlinks/shadow/public1',
],
'../../public2': [
'/tmp/foobaz/symlinks/private-ips/private2',
'/tmp/foobaz/symlinks/zookeeper/public2',
'/tmp/foobaz/symlinks/appengine/public2',
'/tmp/foobaz/symlinks/memcache/public2',
],
'../../public3': [
'/tmp/foobaz/symlinks/private-ips/private3',
'/tmp/foobaz/symlinks/appengine/public3',
'/tmp/foobaz/symlinks/memcache/public3',
]
}
for original_dir, expected_links in links_mapping.iteritems():
for expected_link in expected_links:
os.should_receive('symlink').with_args(original_dir, expected_link)
# finally, fake the copying of the log files
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/var/log/haproxy*'), shell=True, stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/var/log/rabbitmq'), shell=True, stdout=self.fake_temp_file,
stderr=subprocess.STDOUT).and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(
re.compile('/opt/cassandra/cassandra/logs'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT
).and_return(self.success)
argv = [
"--keyname", self.keyname,
"--location", "/tmp/foobaz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.gather_logs(options)
def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive(
'ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args(
'POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args(
'1.23',
'ami-ABCDEFG',
key_name=self.keyname,
network_interfaces=None,
security_groups=[self.group],
instance_type='m3.medium',
count=1,
placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5, stdin='ls').and_return(
'Please login as the user "ubuntu" rather than the user "root"'
)
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'),
None,
5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' ')
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5,
stdin=re.compile('rm -f ')).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), None, 5,
stdin=re.compile('rm -rf ')).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').with_args(
re.compile('scp .*[r|d]sa'), None, 5).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('scp .*{0}'.format(self.keyname)), None,
5).and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(
re.compile('openssl'), None, stdin=None)
self.local_state.should_receive('shell').with_args(
re.compile('^ssh'),
None,
5,
stdin='systemctl start appscale-controller')
self.setup_socket_mocks('public1')
self.setup_appcontroller_mocks('public1', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(
json.loads(
json.dumps([{
"public_ip": "public1",
"private_ip": "private1",
"roles": ["shadow"]
}])))
# copying over the locations json file should be fine
self.local_state.should_receive('shell').with_args(
re.compile('scp'),
None,
5,
stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(
re.compile('scp'),
None,
5,
stdin=re.compile('{0}.secret'.format(self.keyname)))
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
AppControllerClient.should_receive('get_property').\
and_return({'login': '******'})
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance',
state='pending',
key_name=self.keyname,
id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance',
state='running',
key_name=self.keyname,
id='i-ABCDEFG',
ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances') \
.and_return(no_instances) \
.and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1", "--max", "1", "--infrastructure", "ec2",
"--instance_type", "m3.medium", "--machine", "ami-ABCDEFG",
"--use_spot_instances", "--max_spot_price", "1.23", "--keyname",
self.keyname, "--group", self.group, "--test", "--zone",
"my-zone-1b", "--EC2_ACCESS_KEY", "baz", "--EC2_SECRET_KEY", "baz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self):
# let's say that appscale isn't already running
local_appscale_path = os.path.expanduser("~") + os.sep + ".appscale" + \
os.sep + self.keyname + ".key"
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('get_key_path_from_name').and_return(
local_appscale_path)
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').with_args('POST', '/upload', str,
AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').with_args('logs.appscale.com') \
.and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
self.builtins.should_receive('open').with_args(secret_key_location, 'w') \
.and_return(fake_secret)
self.setup_ec2_mocks()
# slip in some fake spot instance info
fake_entry = flexmock(name='fake_entry', price=1)
self.fake_ec2.should_receive('get_spot_price_history').with_args(
start_time=str, end_time=str,
product_description='Linux/UNIX', instance_type='m3.medium',
availability_zone='my-zone-1b').and_return([fake_entry])
# also mock out acquiring a spot instance
self.fake_ec2.should_receive('request_spot_instances').with_args('1.1',
'ami-ABCDEFG', key_name=self.keyname, security_groups=[self.group],
instance_type='m3.medium', count=1, placement='my-zone-1b')
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# assume that root login is not enabled
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin='ls').and_return(
'Please login as the user "ubuntu" rather than the user "root"')
# assume that we can enable root login
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo touch /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin='mktemp').and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sort -u ~/.ssh/authorized_keys /root/.ssh/authorized_keys -o '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5,
stdin=re.compile(
'sudo sed -n '
'\'\/\.\*Please login\/d; w\/root\/\.ssh\/authorized_keys\' '
)
).and_return()
self.local_state.should_receive('shell').with_args(
re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
).and_return()
# and assume that we can copy over our ssh keys fine
self.local_state.should_receive('shell').\
with_args(re.compile('scp .*[r|d]sa'), False, 5).and_return()
self.local_state.should_receive('shell').\
with_args(re.compile('scp .*{0}'.format(self.keyname)), False, 5).\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o '
'NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@public1 ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller '
'/etc/init.d/').\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@elastic-ip ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller '
'/etc/init.d/').\
and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null root@elastic-ip ',
False, 5, stdin='chmod +x /etc/init.d/appcontroller').\
and_return()
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell').with_args(re.compile('openssl'),
False, stdin=None)
# assume that we started monit fine
self.local_state.should_receive('shell').with_args(re.compile('ssh'),
False, 5, stdin=re.compile('monit'))
self.local_state.should_receive('shell').with_args(
re.compile('^ssh'), False, 5, stdin='service appscale-controller start')
self.setup_socket_mocks('elastic-ip')
self.setup_appcontroller_mocks('elastic-ip', 'private1')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "elastic-ip",
"private_ip" : "private1",
"jobs": ["shadow", "login"]
}])))
# copying over the locations yaml and json files should be fine
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('locations-{0}'.format(self.keyname)))
# same for the secret key
self.local_state.should_receive('shell').with_args(re.compile('scp'),
False, 5, stdin=re.compile('{0}.secret'.format(self.keyname)))
flexmock(RemoteHelper).should_receive('copy_deployment_credentials')
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# Let's mock the call to describe_instances when checking for old
# instances to re-use, and then to start the headnode.
pending_instance = flexmock(name='pending_instance', state='pending',
key_name=self.keyname, id='i-ABCDEFG')
pending_reservation = flexmock(name='pending_reservation',
instances=[pending_instance])
no_instances = flexmock(name='no_instances', instances=[])
running_instance = flexmock(name='running_instance', state='running',
key_name=self.keyname, id='i-ABCDEFG', ip_address='public1',
private_ip_address='private1')
running_reservation = flexmock(name='running_reservation',
instances=[running_instance])
self.fake_ec2.should_receive('get_all_instances').and_return(no_instances) \
.and_return(no_instances) \
.and_return(no_instances).and_return(pending_reservation) \
.and_return(running_reservation)
argv = [
"--min", "1",
"--max", "1",
"--infrastructure", "ec2",
"--machine", "ami-ABCDEFG",
"--instance_type", "m3.medium",
"--use_spot_instances",
"--keyname", self.keyname,
"--group", self.group,
"--test",
"--zone", "my-zone-1b",
"--static_ip", "elastic-ip",
"--EC2_ACCESS_KEY", "baz",
"--EC2_SECRET_KEY", "baz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_appscale_in_one_node_virt_deployment(self):
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null [email protected] ",
False, 5, stdin="chmod +x /etc/init.d/appcontroller")
self.local_state.should_receive('shell').\
with_args("ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet "
"-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@public1 ",
False, 5,
stdin="cp /root/appscale/AppController/scripts/appcontroller "
"/etc/init.d/")
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
rh = flexmock(RemoteHelper)
rh.should_receive('copy_deployment_credentials').and_return()
# mock out talking to logs.appscale.com
fake_connection = flexmock(name='fake_connection')
fake_connection.should_receive('request').\
with_args('POST', '/upload', str, AppScaleLogger.HEADERS).and_return()
flexmock(httplib)
httplib.should_receive('HTTPConnection').\
with_args('logs.appscale.com').and_return(fake_connection)
# mock out generating the secret key
flexmock(uuid)
uuid.should_receive('uuid4').and_return('the secret')
# mock out writing the secret key to ~/.appscale, as well as reading it
# later
secret_key_location = LocalState.get_secret_key_location(self.keyname)
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
fake_secret.should_receive('write').and_return()
self.builtins.should_receive('open').\
with_args(secret_key_location, 'r').and_return(fake_secret)
self.builtins.should_receive('open').\
with_args(secret_key_location, 'w').and_return(fake_secret)
# Don't write local metadata files.
flexmock(LocalState).should_receive('update_local_metadata')
# mock out copying over the keys
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.key'),False,5)
self.setup_appscale_compatibility_mocks()
# mock out generating the private key
self.local_state.should_receive('shell')\
.with_args(re.compile('^openssl'),False,stdin=None)\
.and_return()
# mock out removing the old json file
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume that we started monit fine
self.local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('monit'))\
.and_return()
# and that we copied over the AppController's monit file
self.local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
self.local_state.should_receive('shell').\
with_args('ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet '
'-o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no '
'-o UserKnownHostsFile=/dev/null [email protected] ',
False, 5,
stdin='cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/').and_return()
self.setup_socket_mocks('1.2.3.4')
self.setup_appcontroller_mocks('1.2.3.4', '1.2.3.4')
# mock out reading the locations.json file, and slip in our own json
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip": "1.2.3.4",
"private_ip": "1.2.3.4",
"jobs": ["shadow", "login"]
}])))
# Assume the locations files were copied successfully.
locations_file = '{}/locations-bookey.yaml'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_file)), False, 5)\
.and_return()
locations_json = '{}/locations-bookey.json'.\
format(RemoteHelper.CONFIG_DIR)
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(locations_json)), False, 5)\
.and_return()
user_locations = '/root/.appscale/locations-bookey.json'
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*{}'.format(user_locations)), False, 5)\
.and_return()
# Assume the secret key was copied successfully.
self.local_state.should_receive('shell')\
.with_args(re.compile('^scp .*.secret'), False, 5)\
.and_return()
flexmock(AppControllerClient)
AppControllerClient.should_receive('does_user_exist').and_return(True)
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
