def test_copy_deployment_credentials_in_cloud(self):
# mock out the scp'ing to public1 and assume they succeed
local_state = flexmock(LocalState)
local_state.should_receive("shell").with_args(re.compile("^scp .*secret.key"), True, 5).and_return().ordered()
local_state.should_receive("shell").with_args(re.compile("^scp .*ssh.key"), True, 5).and_return().ordered()
# mock out generating the private key
local_state = flexmock(LocalState)
local_state.should_receive("shell").with_args(re.compile("^openssl"), True, stdin=None).and_return().ordered()
local_state.should_receive("shell").with_args(re.compile("^scp .*mycert.pem"), True, 5).and_return().ordered()
local_state.should_receive("shell").with_args(re.compile("^scp .*mykey.pem"), True, 5).and_return().ordered()
# next, mock out copying the private key and certificate
local_state.should_receive("shell").with_args(
re.compile("^ssh"), True, 5, stdin=re.compile("^mkdir -p")
).and_return().ordered()
local_state.should_receive("shell").with_args(
re.compile("^scp .*cloud1/mycert.pem"), True, 5
).and_return().ordered()
local_state.should_receive("shell").with_args(
re.compile("^scp .*cloud1/mykey.pem"), True, 5
).and_return().ordered()
options = flexmock(name="options", keyname="bookey", infrastructure="ec2", verbose=True)
RemoteHelper.copy_deployment_credentials("public1", options)
def test_start_remote_appcontroller(self):
# mock out removing the old json file
local_state = flexmock(LocalState)
local_state.should_receive("shell").with_args(
re.compile("^ssh"), False, 5, stdin=re.compile("rm -rf")
).and_return().ordered()
# assume we started god on public1 fine
local_state.should_receive("shell").with_args(
re.compile("^ssh"), False, 5, stdin=re.compile("nohup god")
).and_return().ordered()
# also assume that we scp'ed over the god config file fine
local_state.should_receive("shell").with_args(
re.compile("scp .*appcontroller\.god.*"), False, 5
).and_return().ordered()
# and assume we started the AppController on public1 fine
local_state.should_receive("shell").with_args(
re.compile("^ssh"), False, 5, stdin=re.compile("^god load .*appcontroller\.god")
).and_return().ordered()
# finally, assume the appcontroller comes up after a few tries
# assume that ssh comes up on the third attempt
fake_socket = flexmock(name="fake_socket")
fake_socket.should_receive("connect").with_args(("public1", AppControllerClient.PORT)).and_raise(
Exception
).and_raise(Exception).and_return(None)
socket.should_receive("socket").and_return(fake_socket)
RemoteHelper.start_remote_appcontroller("public1", "bookey", False)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip, LocalState.get_secret_key(
options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
if not os.path.exists(
LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException(
"AppScale is not running with the keyname {0}".format(
options.keyname))
infrastructure = LocalState.get_infrastructure(options.keyname)
# If the user is on a cloud deployment, and not backing their data to
# persistent disks, warn them before shutting down AppScale.
# Also, if we're in developer mode, skip the warning.
if infrastructure != "xen" and not LocalState.are_disks_used(
options.keyname) and not options.test:
LocalState.ensure_user_wants_to_terminate()
if infrastructure in InfrastructureAgentFactory.VALID_AGENTS:
RemoteHelper.terminate_cloud_infrastructure(
options.keyname, options.verbose)
else:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.verbose)
LocalState.cleanup_appscale_files(options.keyname)
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
def test_wait_for_machines_to_finish_loading(self):
# mock out reading the secret key
builtins = flexmock(sys.modules["__builtin__"])
builtins.should_call("open") # set the fall-through
secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive("read").and_return("the secret")
builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name="fake_soap")
fake_soap.should_receive("get_all_public_ips").with_args("the secret").and_return(
json.dumps(["public1", "public2"])
)
role_info = [
{"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "db_master"]},
{"public_ip": "public2", "private_ip": "private2", "jobs": ["appengine"]},
]
fake_soap.should_receive("get_role_info").with_args("the secret").and_return(json.dumps(role_info))
# also, let's say that our machines aren't running the first time we ask,
# but that they are the second time
fake_soap.should_receive("is_done_initializing").with_args("the secret").and_return(False).and_return(True)
flexmock(SOAPpy)
SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_soap)
SOAPpy.should_receive("SOAPProxy").with_args("https://public2:17443").and_return(fake_soap)
RemoteHelper.wait_for_machines_to_finish_loading("public1", "bookey")
def test_start_remote_appcontroller(self):
# mock out removing the old json file
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume we started monit on public1 fine
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('monit'))\
.and_return()
# also assume that we scp'ed over the god config file fine
local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
# and assume we started the AppController on public1 fine
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'), False, 5,
stdin=re.compile('^monit start -g controller'))\
.and_return()
# finally, assume the appcontroller comes up after a few tries
# assume that ssh comes up on the third attempt
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
AppControllerClient.PORT)).and_raise(Exception) \
.and_raise(Exception).and_return(None)
socket.should_receive('socket').and_return(fake_socket)
RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)
def test_start_remote_appcontroller(self):
# mock out removing the old json file
subprocess.should_receive('Popen').with_args(re.compile('rm -rf'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# assume we started god on public1 fine
subprocess.should_receive('Popen').with_args(re.compile('god &'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# also assume that we scp'ed over the god config file fine
subprocess.should_receive('Popen').with_args(re.compile('appcontroller'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# and assume we started the AppController on public1 fine
subprocess.should_receive('Popen').with_args(re.compile('god load'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# finally, assume the appcontroller comes up after a few tries
# assume that ssh comes up on the third attempt
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
AppControllerClient.PORT)).and_raise(Exception) \
.and_raise(Exception).and_return(None)
socket.should_receive('socket').and_return(fake_socket)
RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)
def test_copy_deployment_credentials_in_cloud(self):
options = flexmock(
keyname='key1',
infrastructure='ec2',
verbose=True,
)
local_state = flexmock(LocalState)
remote_helper = flexmock(RemoteHelper)
local_state.should_receive('get_secret_key_location').and_return()
local_state.should_receive('get_key_path_from_name').and_return()
local_state.should_receive('get_certificate_location').and_return()
local_state.should_receive('get_private_key_location').and_return()
remote_helper.should_receive('scp').and_return()
local_state.should_receive('generate_ssl_cert').and_return()
popen_object = flexmock(communicate=lambda: ['hash_id'])
flexmock(subprocess).should_receive('Popen').and_return(popen_object)
remote_helper.should_receive('ssh').and_return()
flexmock(AppScaleLogger).should_receive('log').and_return()
RemoteHelper.copy_deployment_credentials('public1', options)
options = flexmock(
keyname='key1',
infrastructure='gce',
verbose=True,
)
local_state.should_receive('get_oauth2_storage_location').and_return()
RemoteHelper.copy_deployment_credentials('public1', options)
def testCleanInClusterDeployment(self):
# calling 'appscale clean' in a cluster deployment should ssh into each of
# the boxes specified in the ips_layout and run the terminate script
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout' : {
'controller': 'public1',
'servers': ['public2', 'public3']
},
'test' : True
}
yaml_dumped_contents = yaml.dump(contents)
flexmock(RemoteHelper)
RemoteHelper.should_receive('ssh') \
.with_args(re.compile('public[123]'), 'appscale', str, False)
flexmock(LocalState)
LocalState.should_receive('cleanup_appscale_files').with_args('appscale')
appscale = AppScale()
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
expected = ['public1', 'public2', 'public3']
self.assertEquals(expected, appscale.clean())
def add_instances(cls, options):
"""Adds additional machines to an AppScale deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if 'master' in options.ips.keys():
raise BadConfigurationException("Cannot add master nodes to an " + \
"already running AppScale deployment.")
# Skip checking for -n (replication) because we don't allow the user
# to specify it here (only allowed in run-instances).
additional_nodes_layout = NodeLayout(options)
# In virtualized cluster deployments, we need to make sure that the user
# has already set up SSH keys.
if LocalState.get_from_yaml(options.keyname,
'infrastructure') == "xen":
for ip in options.ips.values():
# throws a ShellException if the SSH key doesn't work
RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)
# Finally, find an AppController and send it a message to add
# the given nodes with the new roles.
AppScaleLogger.log("Sending request to add instances")
login_ip = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_ip,
LocalState.get_secret_key(options.keyname))
acc.start_roles_on_nodes(json.dumps(options.ips))
# TODO(cgb): Should we wait for the new instances to come up and get
# initialized?
AppScaleLogger.success("Successfully sent request to add instances " + \
"to this AppScale deployment.")
def test_copy_deployment_credentials_in_cloud(self):
options = flexmock(
keyname='key1',
infrastructure='ec2',
verbose=True,
)
local_state = flexmock(LocalState)
remote_helper = flexmock(RemoteHelper)
local_state.should_receive('get_secret_key_location').and_return()
local_state.should_receive('get_key_path_from_name').and_return()
local_state.should_receive('get_certificate_location').and_return()
local_state.should_receive('get_private_key_location').and_return()
remote_helper.should_receive('scp').and_return()
local_state.should_receive('generate_ssl_cert').and_return()
popen_object = flexmock(communicate=lambda: ['hash_id'])
flexmock(subprocess).should_receive('Popen').and_return(popen_object)
remote_helper.should_receive('ssh').and_return()
flexmock(AppScaleLogger).should_receive('log').and_return()
RemoteHelper.copy_deployment_credentials('public1', options)
flexmock(GCEAgent).should_receive('get_secrets_type').\
and_return(CredentialTypes.OAUTH)
flexmock(os.path).should_receive('exists').and_return(True)
options = flexmock(
keyname='key1',
infrastructure='gce',
verbose=True,
)
local_state.should_receive('get_oauth2_storage_location').and_return()
RemoteHelper.copy_deployment_credentials('public1', options)
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
if not os.path.exists(
LocalState.get_locations_yaml_location(options.keyname)):
raise AppScaleException(
"AppScale is not running with the keyname {0}".format(
options.keyname))
if LocalState.get_infrastructure(options.keyname) in \
InfrastructureAgentFactory.VALID_AGENTS:
RemoteHelper.terminate_cloud_infrastructure(
options.keyname, options.verbose)
else:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.verbose)
LocalState.cleanup_appscale_files(options.keyname)
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))
infrastructure = LocalState.get_infrastructure(options.keyname)
# If the user is on a cloud deployment, and not backing their data to
# persistent disks, warn them before shutting down AppScale.
# Also, if we're in developer mode, skip the warning.
if infrastructure != "xen" and not LocalState.are_disks_used(options.keyname) and not options.test:
LocalState.ensure_user_wants_to_terminate()
if infrastructure in InfrastructureAgentFactory.VALID_AGENTS:
RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
else:
RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)
LocalState.cleanup_appscale_files(options.keyname)
AppScaleLogger.success("Successfully shut down your AppScale deployment.")
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in acc.get_all_public_ips():
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/appscale', local_dir,
options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def add(self, argv):
parser = argparse.ArgumentParser(usage=self.SUPPORT_CMDS['add'])
parser.add_argument("policy_name")
parser.add_argument("policy_file", nargs="?", type=argparse.FileType('r'), default=sys.stdin)
parser.add_argument("-inactive", action='store_true')
options = parser.parse_args(argv)
content = options.policy_file.read()
if not options.policy_file == sys.stdin:
options.policy_file.close()
if self.eager:
res = self.eager.add_policy(options.policy_name, content, not options.inactive)
if res[0] == 0:
print res[1]
else:
if self.remote_exist(options.policy_name, 'all'):
print "Error: Policy {0} already exists!".format(options.policy_name)
return
if options.inactive:
s = '.i.py'
else:
s = '.a.py'
s = options.policy_name + s
tmp = open(s, "w")
tmp.write(content)
tmp.close()
RemoteHelper.scp(self.service_host, self.key_name, s, self.POLICY_DIR, False)
def testCleanInClusterDeployment(self):
# calling 'appscale clean' in a cluster deployment should ssh into each of
# the boxes specified in the ips_layout and run the terminate script
# Mock out the actual file reading itself, and slip in a YAML-dumped
# file
contents = {
'ips_layout' : {
'controller': 'public1',
'servers': ['public2', 'public3']
},
'test' : True
}
yaml_dumped_contents = yaml.dump(contents)
flexmock(RemoteHelper)
RemoteHelper.should_receive('ssh') \
.with_args(re.compile('public[123]'), 'appscale', str, False)
flexmock(LocalState)
LocalState.should_receive('cleanup_appscale_files').with_args('appscale')
appscale = AppScale()
self.addMockForAppScalefile(appscale, yaml_dumped_contents)
expected = ['public1', 'public2', 'public3']
self.assertEquals(expected, appscale.clean())
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".format(options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException("Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)
except (IOError, AppScaleException):
# Don't fail if we cannot find the configuration.
pass
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate:
RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
def valid_ssh_key(self, config, run_instances_opts):
""" Checks if the tools can log into the head node with the current key.
Args:
config: A dictionary that includes the IPs layout (which itself is a dict
mapping role names to IPs) and, optionally, the keyname to use.
run_instances_opts: The arguments parsed from the appscale-run-instances
command.
Returns:
A bool indicating whether or not the specified keyname can be used to log
into the head node.
Raises:
BadConfigurationException: If the IPs layout was not a dictionary.
"""
keyname = config['keyname']
verbose = config.get('verbose', False)
if not isinstance(config['ips_layout'], dict):
raise BadConfigurationException(
'ips_layout should be a dictionary. Please fix it and try again.'
)
ssh_key_location = self.APPSCALE_DIRECTORY + keyname + ".key"
if not os.path.exists(ssh_key_location):
return False
all_ips = LocalState.get_all_public_ips(keyname)
# If a login node is defined, use that to communicate with other nodes.
node_layout = NodeLayout(run_instances_opts)
head_node = node_layout.head_node()
if head_node is not None:
remote_key = '{}/ssh.key'.format(RemoteHelper.CONFIG_DIR)
try:
RemoteHelper.scp(head_node.public_ip, keyname,
ssh_key_location, remote_key, verbose)
except ShellException:
return False
for ip in all_ips:
ssh_to_ip = 'ssh -i {key} -o StrictHostkeyChecking=no root@{ip} true'\
.format(key=remote_key, ip=ip)
try:
RemoteHelper.ssh(head_node.public_ip,
keyname,
ssh_to_ip,
verbose,
user='******')
except ShellException:
return False
return True
for ip in all_ips:
if not self.can_ssh_to_ip(ip, keyname, verbose):
return False
return True
def test_copy_deployment_credentials_in_cloud(self):
# mock out the scp'ing to public1 and assume they succeed
local_state = flexmock(LocalState)
local_state.should_receive('shell').and_return().ordered()
options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
verbose=True)
RemoteHelper.copy_deployment_credentials('public1', options)
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(
options.keyname, options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, set up passwordless ssh
AppScaleLogger.log(
"Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell(
"{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password),
options.verbose)
else:
LocalState.shell(
"ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
# next, copy over the ssh keypair we generate
RemoteHelper.scp(ip, options.keyname, public_key,
'/root/.ssh/id_rsa.pub', options.verbose)
RemoteHelper.scp(ip, options.keyname, private_key,
'/root/.ssh/id_rsa', options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def valid_ssh_key(self, config, run_instances_opts):
""" Checks if the tools can log into the head node with the current key.
Args:
config: A dictionary that includes the IPs layout (which itself is a dict
mapping role names to IPs) and, optionally, the keyname to use.
run_instances_opts: The arguments parsed from the appscale-run-instances
command.
Returns:
A bool indicating whether or not the specified keyname can be used to log
into the head node.
Raises:
BadConfigurationException: If the IPs layout was not a dictionary.
"""
keyname = config['keyname']
verbose = config.get('verbose', False)
if not isinstance(config['ips_layout'], dict):
raise BadConfigurationException(
'ips_layout should be a dictionary. Please fix it and try again.')
ssh_key_location = self.APPSCALE_DIRECTORY + keyname + ".key"
if not os.path.exists(ssh_key_location):
return False
all_ips = LocalState.get_all_public_ips(keyname)
# If a login node is defined, use that to communicate with other nodes.
node_layout = NodeLayout(run_instances_opts)
head_node = node_layout.head_node()
if head_node is not None:
remote_key = '{}/ssh.key'.format(RemoteHelper.CONFIG_DIR)
try:
RemoteHelper.scp(
head_node.public_ip, keyname, ssh_key_location, remote_key, verbose)
except ShellException:
return False
for ip in all_ips:
ssh_to_ip = 'ssh -i {key} -o StrictHostkeyChecking=no root@{ip} true'\
.format(key=remote_key, ip=ip)
try:
RemoteHelper.ssh(
head_node.public_ip, keyname, ssh_to_ip, verbose, user='******')
except ShellException:
return False
return True
for ip in all_ips:
if not self.can_ssh_to_ip(ip, keyname, verbose):
return False
return True
def test_rsync_files_from_dir_that_does_exist(self):
# if the user specifies that we should copy from a directory that does
# exist, and has all the right directories in it, we should succeed
flexmock(os.path)
os.path.should_receive("exists").with_args(re.compile("/tmp/booscale-local/")).and_return(True)
# assume the rsyncs succeed
local_state = flexmock(LocalState)
local_state.should_receive("shell").with_args(re.compile("^rsync"), False).and_return().ordered()
RemoteHelper.rsync_files("public1", "booscale", "/tmp/booscale-local", False)
def run_bootstrap(cls, ip, options, error_ips):
try:
RemoteHelper.ssh(ip, options.keyname, cls.BOOTSTRAP_CMD, options.verbose)
AppScaleLogger.success(
'Successfully updated and built AppScale on {}'.format(ip))
except ShellException:
error_ips.append(ip)
AppScaleLogger.warn('Unable to upgrade AppScale code on {}.\n'
'Please correct any errors listed in /var/log/appscale/bootstrap.log '
'on that machine and re-run appscale upgrade.'.format(ip))
return error_ips
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
local_dir, options.verbose)
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/cassandra', local_dir, options.verbose)
except ShellException:
pass
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
'/var/log/zookeeper', local_dir, options.verbose)
except ShellException:
pass
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/kern.log',
local_dir, options.verbose)
RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/syslog',
local_dir, options.verbose)
AppScaleLogger.success("Successfully copied logs to {0}".format(
options.location))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
# next, copy over the ssh keypair we generate
RemoteHelper.scp(ip, options.keyname, public_key, '/root/.ssh/id_rsa.pub',
options.verbose)
RemoteHelper.scp(ip, options.keyname, private_key, '/root/.ssh/id_rsa',
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def test_start_remote_appcontroller(self):
# mock out removing the old json file
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
.and_return()
# assume we started monit on public1 fine
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('monit'))\
.and_return()
# also assume that we scp'ed over the god config file fine
local_state.should_receive('shell')\
.with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
.and_return()
# and assume we started the AppController on public1 fine
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'), False, 5,
stdin=re.compile('^monit start -g controller'))\
.and_return()
# finally, assume the appcontroller comes up after a few tries
# assume that ssh comes up on the third attempt
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
AppControllerClient.PORT)).and_raise(Exception) \
.and_raise(Exception).and_return(None)
socket.should_receive('socket').and_return(fake_socket)
# Mock out additional remote calls.
local_state.should_receive('shell').with_args(
'ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ',
False,
5,
stdin=
'cp /root/appscale/AppController/scripts/appcontroller /etc/init.d/'
).and_return()
local_state.should_receive('shell').with_args(
'ssh -i /root/.appscale/bookey.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@public1 ',
False,
5,
stdin='chmod +x /etc/init.d/appcontroller').and_return()
local_state.should_receive('shell').with_args(
'ssh -i /root/.appscale/boobazblargfoo.key -o LogLevel=quiet -o NumberOfPasswordPrompts=0 -o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null root@elastic-ip ',
False,
5,
stdin='chmod +x /etc/init.d/appcontroller').and_return()
RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)
def run_bootstrap(cls, ip, options, error_ips):
try:
RemoteHelper.ssh(ip, options.keyname, cls.BOOTSTRAP_CMD, options.verbose)
AppScaleLogger.success("Successfully updated and built AppScale on {}".format(ip))
except ShellException:
error_ips.append(ip)
AppScaleLogger.warn(
"Unable to upgrade AppScale code on {}.\n"
"Please correct any errors listed in /var/log/appscale/bootstrap.log "
"on that machine and re-run appscale upgrade.".format(ip)
)
return error_ips
def test_copy_deployment_credentials_in_cloud(self):
# mock out the scp'ing to public1 and assume they succeed
subprocess.should_receive('Popen').with_args(re.compile('secret.key'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('ssh.key'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# mock out generating the private key
flexmock(M2Crypto.RSA)
fake_rsa_key = flexmock(name='fake_rsa_key')
fake_rsa_key.should_receive('save_key').with_args(
LocalState.get_private_key_location('bookey'), None)
M2Crypto.RSA.should_receive('gen_key').and_return(fake_rsa_key)
flexmock(M2Crypto.EVP)
fake_pkey = flexmock(name='fake_pkey')
fake_pkey.should_receive('assign_rsa').with_args(fake_rsa_key).and_return()
M2Crypto.EVP.should_receive('PKey').and_return(fake_pkey)
# and mock out generating the certificate
flexmock(M2Crypto.X509)
fake_cert = flexmock(name='fake_x509')
fake_cert.should_receive('set_pubkey').with_args(fake_pkey).and_return()
fake_cert.should_receive('set_subject')
fake_cert.should_receive('set_issuer_name')
fake_cert.should_receive('set_not_before')
fake_cert.should_receive('set_not_after')
fake_cert.should_receive('sign').with_args(fake_pkey, md="sha256")
fake_cert.should_receive('save_pem').with_args(
LocalState.get_certificate_location('bookey'))
M2Crypto.X509.should_receive('X509').and_return(fake_cert)
# next, mock out copying the private key and certificate
subprocess.should_receive('Popen').with_args(re.compile('mycert.pem'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('mykey.pem'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
verbose=True)
RemoteHelper.copy_deployment_credentials('public1', options)
def async_layout_upgrade(ip, keyname, script, error_bucket, verbose=False):
""" Run a command over SSH and place exceptions in a bucket.
Args:
ip: A string containing and IP address.
keyname: A string containing the deployment keyname.
script: A string to run as a command over SSH.
error_bucket: A thread-safe queue.
verbose: A boolean indicating whether or not to log verbosely.
"""
try:
RemoteHelper.ssh(ip, keyname, script, verbose)
except ShellException as ssh_error:
error_bucket.put(ssh_error)
def async_layout_upgrade(ip, keyname, script, error_bucket, verbose=False):
""" Run a command over SSH and place exceptions in a bucket.
Args:
ip: A string containing and IP address.
keyname: A string containing the deployment keyname.
script: A string to run as a command over SSH.
error_bucket: A thread-safe queue.
verbose: A boolean indicating whether or not to log verbosely.
"""
try:
RemoteHelper.ssh(ip, keyname, script, verbose)
except ShellException as ssh_error:
error_bucket.put(ssh_error)
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
self.local_state.should_receive('ensure_appscale_isnt_running').and_return()
self.local_state.should_receive('make_appscale_directory').and_return()
self.local_state.should_receive('update_local_metadata').and_return()
self.local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "1.2.3.4",
"private_ip" : "1.2.3.4",
"jobs" : ["shadow", "login"]
}])))
self.local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('start_head_node')\
.and_return(('1.2.3.4','i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
acc = flexmock(AppControllerClient)
acc.should_receive('is_initialized').and_return(True)
uac = flexmock(UserAppClient)
uac.should_receive('does_user_exist').and_return(False)
flexmock(UserAppClient).should_receive('set_admin_role').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test",
"--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_copy_local_metadata(self):
# mock out the copying of the two files
subprocess.should_receive('Popen').with_args(re.compile(
'locations-bookey.[yaml|json]'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# and mock out copying the secret file
subprocess.should_receive('Popen').with_args(re.compile(
'bookey.secret'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
RemoteHelper.copy_local_metadata('public1', 'bookey', False)
def test_rsync_files_from_dir_that_does_exist(self):
# if the user specifies that we should copy from a directory that does
# exist, and has all the right directories in it, we should succeed
flexmock(os.path)
os.path.should_receive('exists').with_args(re.compile(
'/tmp/booscale-local/')).and_return(True)
# assume the rsyncs succeed
subprocess.should_receive('Popen').with_args(re.compile('rsync'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
RemoteHelper.rsync_files('public1', 'booscale', '/tmp/booscale-local',
False)
def test_appscale_in_one_node_virt_deployment_with_login_override(self):
# let's say that appscale isn't already running
local_state = flexmock(LocalState)
local_state.should_receive('ensure_appscale_isnt_running').and_return()
local_state.should_receive('make_appscale_directory').and_return()
local_state.should_receive('update_local_metadata').and_return()
local_state.should_receive('get_local_nodes_info').and_return(json.loads(
json.dumps([{
"public_ip" : "1.2.3.4",
"private_ip" : "1.2.3.4",
"jobs" : ["shadow", "login"]
}])))
local_state.should_receive('get_secret_key').and_return("fookey")
flexmock(RemoteHelper)
RemoteHelper.should_receive('start_head_node')\
.and_return(('1.2.3.4','i-ABCDEFG'))
RemoteHelper.should_receive('sleep_until_port_is_open').and_return()
RemoteHelper.should_receive('copy_local_metadata').and_return()
RemoteHelper.should_receive('create_user_accounts').and_return()
RemoteHelper.should_receive('wait_for_machines_to_finish_loading')\
.and_return()
acc = flexmock(AppControllerClient)
acc.should_receive('get_uaserver_host').and_return('host')
flexmock(UserAppClient).should_receive('set_admin_role').and_return()
# don't use a 192.168.X.Y IP here, since sometimes we set our virtual
# machines to boot with those addresses (and that can mess up our tests).
ips_layout = yaml.safe_load("""
master : 1.2.3.4
database: 1.2.3.4
zookeeper: 1.2.3.4
appengine: 1.2.3.4
""")
argv = [
"--ips_layout", base64.b64encode(yaml.dump(ips_layout)),
"--keyname", self.keyname,
"--test",
"--login_host", "www.booscale.com"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.run_instances(options)
def test_rsync_files_from_dir_that_does_exist(self):
# if the user specifies that we should copy from a directory that does
# exist, and has all the right directories in it, we should succeed
flexmock(os.path)
os.path.should_receive('exists').with_args(re.compile(
'/tmp/booscale-local/')).and_return(True)
# assume the rsyncs succeed
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('^rsync'),False)\
.and_return().ordered()
RemoteHelper.rsync_files('public1', 'booscale', '/tmp/booscale-local',
False)
def test_copy_local_metadata(self):
# mock out the copying of the two files
local_state = flexmock(LocalState)
local_state.should_receive('shell').with_args(
re.compile('^scp .*/etc/appscale/locations-bookey.yaml'), False, 5)
local_state.should_receive('shell').with_args(
re.compile('^scp .*/etc/appscale/locations-bookey.json'), False, 5)
local_state.should_receive('shell').with_args(
re.compile('^scp .*/root/.appscale/locations-bookey.json'), False, 5)
# and mock out copying the secret file
local_state.should_receive('shell').with_args(
re.compile('^scp .*bookey.secret'), False, 5)
RemoteHelper.copy_local_metadata('public1', 'bookey', False)
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def test_create_user_accounts(self):
# mock out reading the secret key
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location('bookey')).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(
json.dumps([{
"public_ip": "public1",
"private_ip": "private1",
"jobs": ["shadow", "login"]
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location('bookey'), 'r') \
.and_return(fake_nodes_json)
# mock out SOAP interactions with the UserAppServer
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('false')
fake_soap.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_soap.should_receive('does_user_exist').with_args(
'boo@public1', 'the secret').and_return('false')
fake_soap.should_receive('commit_new_user').with_args(
'boo@public1', str, 'xmpp_user', 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', 'password', 'public1',
'bookey', False)
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException(
"Cannot find AppScale's configuration for keyname {0}".format(
options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException(
"Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(
LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException(
"AppScale is not running with the keyname {0}".format(
options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.clean,
options.verbose)
except (IOError, AppScaleException):
# Don't fail if we cannot find the configuration.
pass
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS
and options.terminate):
RemoteHelper.terminate_cloud_infrastructure(
options.keyname, options.verbose)
if options.clean:
LocalState.clean_local_metadata(keyname=options.keyname)
def clean(self):
"""'clean' provides a mechanism that will forcefully shut down all AppScale-
related services on virtual machines in a cluster deployment.
Returns:
A list of the IP addresses where AppScale was shut down.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
BadConfigurationException: If this method is invoked and the AppScalefile
indicates that a cloud deployment is being used.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
if 'ips_layout' not in contents_as_yaml:
raise BadConfigurationException("Cannot use 'appscale clean' in a " \
"cloud deployment.")
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
is_verbose = contents_as_yaml['verbose']
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = 'appscale'
all_ips = self.get_all_ips(contents_as_yaml["ips_layout"])
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
try:
LocalState.cleanup_appscale_files(keyname)
except Exception:
pass
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
return all_ips
def test_create_user_accounts(self):
# mock out reading the secret key
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out reading the locations.json file, and slip in our own json
flexmock(os.path)
os.path.should_call('exists') # set the fall-through
os.path.should_receive('exists').with_args(
LocalState.get_locations_json_location('bookey')).and_return(True)
fake_nodes_json = flexmock(name="fake_nodes_json")
fake_nodes_json.should_receive('read').and_return(json.dumps([{
"public_ip" : "public1",
"private_ip" : "private1",
"jobs" : ["shadow", "login"]
}]))
builtins.should_receive('open').with_args(
LocalState.get_locations_json_location('bookey'), 'r') \
.and_return(fake_nodes_json)
# mock out SOAP interactions with the UserAppServer
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('does_user_exist').with_args('*****@*****.**',
'the secret').and_return('false')
fake_soap.should_receive('commit_new_user').with_args('*****@*****.**', str,
'xmpp_user', 'the secret').and_return('true')
fake_soap.should_receive('does_user_exist').with_args('boo@public1',
'the secret').and_return('false')
fake_soap.should_receive('commit_new_user').with_args('boo@public1', str,
'xmpp_user', 'the secret').and_return('true')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@foo.goo', 'password', 'public1',
'bookey', False)
def test_copy_deployment_credentials_in_cloud(self):
# mock out the scp'ing to public1 and assume they succeed
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*secret.key'),True,5)\
.and_return().ordered()
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*ssh.key'),True,5)\
.and_return().ordered()
# mock out generating the private key
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('^openssl'),True, stdin=None)\
.and_return().ordered()
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*mycert.pem'),True,5)\
.and_return().ordered()
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*mykey.pem'),True,5)\
.and_return().ordered()
# next, mock out copying the private key and certificate
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),True,5,stdin=re.compile('^mkdir -p'))\
.and_return().ordered()
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*cloud1/mycert.pem'),True,5)\
.and_return().ordered()
local_state.should_receive('shell')\
.with_args(re.compile('^scp .*cloud1/mykey.pem'),True,5)\
.and_return().ordered()
options = flexmock(name='options',
keyname='bookey',
infrastructure='ec2',
verbose=True)
RemoteHelper.copy_deployment_credentials('public1', options)
def remove_app(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input(
'Are you sure you want to remove this application? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled application removal.")
login_host = LocalState.get_login_host(options.keyname)
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret)
if not acc.is_app_running(options.appname):
raise AppScaleException("The given application is not currently running.")
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
http_port = None
for _ in range(cls.MAX_RETRIES + 1):
result = acc.get_all_stats()
try:
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[options.appname]
http_port = current_app['http']
if http_port:
break
time.sleep(cls.SLEEP_TIME)
except (KeyError, ValueError):
AppScaleLogger.verbose("Got json error from get_all_data result.",
options.verbose)
time.sleep(cls.SLEEP_TIME)
if not http_port:
raise AppScaleException(
"Unable to get the serving port for the application.")
acc.stop_app(options.appname)
AppScaleLogger.log("Please wait for your app to shut down.")
for _ in range(cls.MAX_RETRIES + 1):
if RemoteHelper.is_port_open(login_host, http_port, options.verbose):
time.sleep(cls.SLEEP_TIME)
AppScaleLogger.log("Waiting for {0} to terminate...".format(
options.appname))
else:
AppScaleLogger.success("Done shutting down {0}.".format(
options.appname))
return
AppScaleLogger.warn("App {0} may still be running.".format(
options.appname))
def clean(self):
"""'clean' provides a mechanism that will forcefully shut down all AppScale-
related services on virtual machines in a cluster deployment.
Returns:
A list of the IP addresses where AppScale was shut down.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
BadConfigurationException: If this method is invoked and the AppScalefile
indicates that a cloud deployment is being used.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
if 'ips_layout' not in contents_as_yaml:
raise BadConfigurationException("Cannot use 'appscale clean' in a " \
"cloud deployment.")
if 'verbose' in contents_as_yaml and contents_as_yaml['verbose'] == True:
is_verbose = contents_as_yaml['verbose']
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = 'appscale'
all_ips = self.get_all_ips(contents_as_yaml["ips_layout"])
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
try:
LocalState.cleanup_appscale_files(keyname)
except Exception:
pass
AppScaleLogger.success("Successfully shut down your AppScale deployment.")
return all_ips
def test_wait_for_machines_to_finish_loading(self):
# mock out reading the secret key
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
fake_secret = flexmock(name="fake_secret")
fake_secret.should_receive('read').and_return('the secret')
builtins.should_receive('open').with_args(secret_key_location, 'r') \
.and_return(fake_secret)
# mock out getting all the ips in the deployment from the head node
fake_soap = flexmock(name='fake_soap')
fake_soap.should_receive('get_all_public_ips').with_args('the secret') \
.and_return(json.dumps(['public1', 'public2']))
role_info = [
{
'public_ip' : 'public1',
'private_ip' : 'private1',
'jobs' : ['shadow', 'db_master']
},
{
'public_ip' : 'public2',
'private_ip' : 'private2',
'jobs' : ['appengine']
}
]
fake_soap.should_receive('get_role_info').with_args('the secret') \
.and_return(json.dumps(role_info))
# also, let's say that our machines aren't running the first time we ask,
# but that they are the second time
fake_soap.should_receive('is_done_initializing').with_args('the secret') \
.and_return(False).and_return(True)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
.and_return(fake_soap)
SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
.and_return(fake_soap)
RemoteHelper.wait_for_machines_to_finish_loading('public1', 'bookey')
def can_ssh_to_ip(self, ip, keyname, is_verbose):
"""Attempts to SSH into the machine located at the given IP address with the
given SSH key.
Args:
ip: The IP address to attempt to SSH into.
keyname: The name of the SSH key that uniquely identifies this AppScale
deployment.
is_verbose: A bool that indicates if we should print the SSH command we
execute to stdout.
Returns:
A bool that indicates whether or not the given SSH key can log in without
a password to the given machine.
"""
try:
RemoteHelper.ssh(ip, keyname, 'ls', is_verbose, user='******')
return True
except ShellException:
return False
def can_ssh_to_ip(self, ip, keyname, is_verbose):
""" Attempts to SSH into the machine located at the given IP address with the
given SSH key.
Args:
ip: The IP address to attempt to SSH into.
keyname: The name of the SSH key that uniquely identifies this AppScale
deployment.
is_verbose: A bool that indicates if we should print the SSH command we
execute to stdout.
Returns:
A bool that indicates whether or not the given SSH key can log in without
a password to the given machine.
"""
try:
RemoteHelper.ssh(ip, keyname, 'ls', is_verbose, user='******')
return True
except ShellException:
return False
def upgrade(cls, options):
""" Upgrades the deployment to the latest AppScale version.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException(
'Your ips_layout is invalid:\n{}'.format(node_layout.errors()))
latest_tools = APPSCALE_VERSION
try:
AppScaleLogger.log(
'Checking if an update is available for appscale-tools')
latest_tools = latest_tools_version()
except:
# Prompt the user if version metadata can't be fetched.
if not options.test:
response = raw_input(
'Unable to check for the latest version of appscale-tools. Would '
'you like to continue upgrading anyway? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException('Cancelled AppScale upgrade.')
if latest_tools > APPSCALE_VERSION:
raise AppScaleException(
"There is a newer version ({}) of appscale-tools available. Please "
"upgrade the tools package before running 'appscale upgrade'.".
format(latest_tools))
master_ip = node_layout.head_node().public_ip
upgrade_version_available = cls.get_upgrade_version_available()
current_version = RemoteHelper.get_host_appscale_version(
master_ip, options.keyname, options.verbose)
# Don't run bootstrap if current version is later that the most recent
# public one. Covers cases of revoked versions/tags and ensures we won't
# try to downgrade the code.
if current_version >= upgrade_version_available:
AppScaleLogger.log(
'AppScale is already up to date. Skipping code upgrade.')
AppScaleLogger.log(
'Running upgrade script to check if any other upgrades are needed.'
)
cls.shut_down_appscale_if_running(options)
cls.run_upgrade_script(options, node_layout)
return
cls.shut_down_appscale_if_running(options)
cls.upgrade_appscale(options, node_layout)
def relocate_app(cls, options):
"""Instructs AppScale to move the named application to a different port.
Args:
options: A Namespace that has fields for each parameter that can be passed
in via the command-line interface.
Raises:
AppScaleException: If the named application isn't running in this AppScale
cloud, if the destination port is in use by a different application, or
if the AppController rejects the request to relocate the application (in
which case it includes the reason why the rejection occurred).
"""
login_host = LocalState.get_login_host(options.keyname)
acc = AppControllerClient(login_host,
LocalState.get_secret_key(options.keyname))
app_info_map = acc.get_app_info_map()
if options.appname not in app_info_map.keys():
raise AppScaleException("The given application, {0}, is not currently " \
"running in this AppScale cloud, so we can't move it to a different " \
"port.".format(options.appname))
relocate_result = acc.relocate_app(options.appname, options.http_port,
options.https_port)
if relocate_result == "OK":
AppScaleLogger.success("Successfully issued request to move {0} to " \
"ports {1} and {2}.".format(options.appname, options.http_port,
options.https_port))
RemoteHelper.sleep_until_port_is_open(login_host,
options.http_port,
options.verbose)
AppScaleLogger.success(
"Your app serves unencrypted traffic at: " +
"http://{0}:{1}".format(login_host, options.http_port))
AppScaleLogger.success(
"Your app serves encrypted traffic at: " +
"https://{0}:{1}".format(login_host, options.https_port))
else:
raise AppScaleException(relocate_result)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(
options.file, options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(
options.file, options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException as config_error:
AppScaleLogger.log(config_error)
if 'yaml' in str(config_error):
raise config_error
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == 'java':
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn(
'AppScale did not find the correct SDK jar ' +
'versions in your app. The current supported ' +
'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION +
'.')
login_host = LocalState.get_login_host(options.keyname)
secret_key = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret_key)
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not acc.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username,
password,
login_host,
options.keyname,
clear_datastore=False)
app_exists = acc.does_app_exist(app_id)
app_admin = acc.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
if app_exists:
AppScaleLogger.log(
"Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log(
"Uploading initial version of app {0}".format(app_id))
acc.reserve_app_id(username, app_id, app_language)
# Ignore all .pyc files while tarring.
if app_language == 'python27':
AppScaleLogger.log("Ignoring .pyc files")
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
sleep_time = 2 * cls.SLEEP_TIME
current_app = None
for i in range(cls.MAX_RETRIES):
try:
result = acc.get_all_stats()
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[app_id]
http_port = current_app['http']
break
except ValueError:
pass
except KeyError:
pass
AppScaleLogger.verbose("Waiting {0} second(s) for a port to be assigned to {1}".\
format(sleep_time, app_id), options.verbose)
time.sleep(sleep_time)
if not current_app:
raise AppScaleException(
"Unable to get the serving port for the application.")
RemoteHelper.sleep_until_port_is_open(login_host, http_port,
options.verbose)
AppScaleLogger.success(
"Your app can be reached at the following URL: " +
"http://{0}:{1}".format(login_host, http_port))
if created_dir:
shutil.rmtree(file_location)
return (login_host, http_port)
def run_instances(cls, options):
"""Starts a new AppScale deployment with the parameters given.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppControllerException: If the AppController on the head node crashes.
When this occurs, the message in the exception contains the reason why
the AppController crashed.
BadConfigurationException: If the user passes in options that are not
sufficient to start an AppScale deployment (e.g., running on EC2 but
not specifying the AMI to use), or if the user provides us
contradictory options (e.g., running on EC2 but not specifying EC2
credentials).
"""
LocalState.make_appscale_directory()
LocalState.ensure_appscale_isnt_running(options.keyname, options.force)
if options.infrastructure:
if not options.disks and not options.test and not options.force:
LocalState.ensure_user_wants_to_run_without_disks()
AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
" over the " + options.infrastructure +
" cloud.")
else:
AppScaleLogger.log("Starting AppScale " + APPSCALE_VERSION +
" over a virtualized cluster.")
my_id = str(uuid.uuid4())
AppScaleLogger.remote_log_tools_state(options, my_id, "started",
APPSCALE_VERSION)
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were errors with your " + \
"placement strategy:\n{0}".format(str(node_layout.errors())))
public_ip, instance_id = RemoteHelper.start_head_node(
options, my_id, node_layout)
AppScaleLogger.log(
"\nPlease wait for AppScale to prepare your machines " +
"for use. This can take few minutes.")
# Write our metadata as soon as possible to let users SSH into those
# machines via 'appscale ssh'.
LocalState.update_local_metadata(options, node_layout, public_ip,
instance_id)
RemoteHelper.copy_local_metadata(public_ip, options.keyname,
options.verbose)
acc = AppControllerClient(public_ip,
LocalState.get_secret_key(options.keyname))
# Let's now wait till the server is initialized.
while not acc.is_initialized():
AppScaleLogger.log('Waiting for head node to initialize...')
# This can take some time in particular the first time around, since
# we will have to initialize the database.
time.sleep(cls.SLEEP_TIME * 3)
try:
# We don't need to have any exception information here: we do expect
# some anyway while the UserAppServer is coming up.
acc.does_user_exist("non-existent-user", True)
except Exception as exception:
AppScaleLogger.log('UserAppServer not ready yet. Retrying ...')
time.sleep(cls.SLEEP_TIME)
# Update our metadata again so that users can SSH into other boxes that
# may have been started.
LocalState.update_local_metadata(options, node_layout, public_ip,
instance_id)
RemoteHelper.copy_local_metadata(public_ip, options.keyname,
options.verbose)
if options.admin_user and options.admin_pass:
AppScaleLogger.log("Using the provided admin username/password")
username, password = options.admin_user, options.admin_pass
elif options.test:
AppScaleLogger.log("Using default admin username/password")
username, password = LocalState.DEFAULT_USER, LocalState.DEFAULT_PASSWORD
else:
username, password = LocalState.get_credentials()
RemoteHelper.create_user_accounts(username, password, public_ip,
options.keyname,
options.clear_datastore)
acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES)
RemoteHelper.wait_for_machines_to_finish_loading(
public_ip, options.keyname)
# Finally, update our metadata once we know that all of the machines are
# up and have started all their API services.
LocalState.update_local_metadata(options, node_layout, public_ip,
instance_id)
RemoteHelper.copy_local_metadata(public_ip, options.keyname,
options.verbose)
RemoteHelper.sleep_until_port_is_open(
LocalState.get_login_host(options.keyname),
RemoteHelper.APP_DASHBOARD_PORT, options.verbose)
AppScaleLogger.success("AppScale successfully started!")
AppScaleLogger.success("View status information about your AppScale " + \
"deployment at http://{0}:{1}/status".format(LocalState.get_login_host(
options.keyname), RemoteHelper.APP_DASHBOARD_PORT))
AppScaleLogger.remote_log_tools_state(options, my_id, "finished",
APPSCALE_VERSION)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(options.location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {0}, already exists.".format(options.location))
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(options.location)
# The log paths that we collect logs from.
log_paths = [
'/var/log/appscale', '/var/log/kern.log*', '/var/log/monit.log*',
'/var/log/nginx', '/var/log/syslog*', '/var/log/zookeeper'
]
failures = False
for ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = "{0}/{1}".format(options.location, ip)
os.mkdir(local_dir)
for log_path in log_paths:
try:
RemoteHelper.scp_remote_to_local(ip, options.keyname,
log_path, local_dir,
options.verbose)
except ShellException as shell_exception:
failures = True
AppScaleLogger.warn(
"Unable to collect logs from '{}' for host '{}'".
format(log_path, ip))
AppScaleLogger.verbose(
"Encountered exception: {}".format(
str(shell_exception)), options.verbose)
if failures:
AppScaleLogger.log(
"Done copying to {0}. There were "
"failures while collecting AppScale logs.".format(
options.location))
else:
AppScaleLogger.success(
"Successfully collected all AppScale logs into "
"{0}".format(options.location))
def down(self, clean=False, terminate=False):
""" 'down' provides a nicer experience for users than the
appscale-terminate-instances command, by using the configuration options
present in the AppScalefile found in the current working directory.
Args:
clean: A boolean to indicate if the deployment data and metadata
needs to be clean. This will clear the datastore.
terminate: A boolean to indicate if instances needs to be terminated
(valid only if we spawn instances at start).
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
"""
contents = self.read_appscalefile()
# Construct a terminate-instances command from the file's contents
command = []
contents_as_yaml = yaml.safe_load(contents)
if 'verbose' in contents_as_yaml and contents_as_yaml[
'verbose'] == True:
is_verbose = contents_as_yaml['verbose']
command.append("--verbose")
else:
is_verbose = False
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
command.append("--keyname")
command.append(contents_as_yaml['keyname'])
else:
keyname = 'appscale'
if "EC2_ACCESS_KEY" in contents_as_yaml:
os.environ["EC2_ACCESS_KEY"] = contents_as_yaml["EC2_ACCESS_KEY"]
if "EC2_SECRET_KEY" in contents_as_yaml:
os.environ["EC2_SECRET_KEY"] = contents_as_yaml["EC2_SECRET_KEY"]
if "EC2_URL" in contents_as_yaml:
os.environ["EC2_URL"] = contents_as_yaml["EC2_URL"]
if clean:
if 'test' not in contents_as_yaml or contents_as_yaml[
'test'] != True:
LocalState.confirm_or_abort(
"Clean will delete every data in the deployment.")
all_ips = LocalState.get_all_public_ips(keyname)
for ip in all_ips:
RemoteHelper.ssh(ip, keyname, self.TERMINATE, is_verbose)
AppScaleLogger.success(
"Successfully cleaned your AppScale deployment.")
if terminate:
infrastructure = LocalState.get_infrastructure(keyname)
if infrastructure != "xen" and not LocalState.are_disks_used(
keyname) and 'test' not in contents_as_yaml:
LocalState.confirm_or_abort(
"Terminate will delete instances and the data on them.")
command.append("--terminate")
if 'test' in contents_as_yaml and contents_as_yaml['test'] == True:
command.append("--test")
# Finally, exec the command. Don't worry about validating it -
# appscale-terminate-instances will do that for us.
options = ParseArgs(command, "appscale-terminate-instances").args
AppScaleTools.terminate_instances(options)
LocalState.cleanup_appscale_files(keyname, terminate)
AppScaleLogger.success(
"Successfully shut down your AppScale deployment.")
def run_upgrade_script(cls, options, node_layout):
""" Runs the upgrade script which checks for any upgrades needed to be performed.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
node_layout: A NodeLayout object for the deployment.
"""
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
db_ips = [node.private_ip for node in node_layout.nodes
if node.is_role('db_master') or node.is_role('db_slave')]
zk_ips = [node.private_ip for node in node_layout.nodes
if node.is_role('zookeeper')]
upgrade_script_command = '{script} --keyname {keyname} '\
'--log-postfix {timestamp} '\
'--db-master {db_master} '\
'--zookeeper {zk_ips} '\
'--database {db_ips} '\
'--replication {replication}'.format(
script=cls.UPGRADE_SCRIPT,
keyname=options.keyname,
timestamp=timestamp,
db_master=node_layout.db_master().private_ip,
zk_ips=' '.join(zk_ips),
db_ips=' '.join(db_ips),
replication=node_layout.replication
)
master_public_ip = node_layout.head_node().public_ip
AppScaleLogger.log("Running upgrade script to check if any other upgrade is needed.")
# Run the upgrade command as a background process.
error_bucket = Queue.Queue()
threading.Thread(
target=async_layout_upgrade,
args=(master_public_ip, options.keyname, upgrade_script_command,
error_bucket, options.verbose)
).start()
last_message = None
while True:
# Check if the SSH thread has crashed.
try:
ssh_error = error_bucket.get(block=False)
AppScaleLogger.warn('Error executing upgrade script')
LocalState.generate_crash_log(ssh_error, traceback.format_exc())
except Queue.Empty:
pass
upgrade_status_file = cls.UPGRADE_STATUS_FILE_LOC + timestamp + ".json"
command = 'cat' + " " + upgrade_status_file
upgrade_status = RemoteHelper.ssh(
master_public_ip, options.keyname, command, options.verbose)
json_status = json.loads(upgrade_status)
if 'status' not in json_status or 'message' not in json_status:
raise AppScaleException('Invalid status log format')
if json_status['status'] == 'complete':
AppScaleLogger.success(json_status['message'])
break
if json_status['status'] == 'inProgress':
if json_status['message'] != last_message:
AppScaleLogger.log(json_status['message'])
last_message = json_status['message']
time.sleep(cls.SLEEP_TIME)
continue
# Assume the message is an error.
AppScaleLogger.warn(json_status['message'])
raise AppScaleException(json_status['message'])
