def update_dispatch(cls, source_location, keyname, project_id):
""" Updates an application's dispatch routing rules from the configuration
file.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if cls.TAR_GZ_REGEX.search(source_location):
fetch_function = utils.config_from_tar_gz
version = Version.from_tar_gz(source_location)
elif cls.ZIP_REGEX.search(source_location):
fetch_function = utils.config_from_zip
version = Version.from_zip(source_location)
elif os.path.isdir(source_location):
fetch_function = utils.config_from_dir
version = Version.from_directory(source_location)
elif source_location.endswith('.yaml'):
fetch_function = utils.config_from_dir
version = Version.from_yaml_file(source_location)
source_location = os.path.dirname(source_location)
else:
raise BadConfigurationException(
'{} must be a directory, tar.gz, or zip'.format(source_location))
if project_id:
version.project_id = project_id
dispatch_rules = utils.dispatch_from_yaml(source_location, fetch_function)
if dispatch_rules is None:
return
AppScaleLogger.log('Updating dispatch for {}'.format(version.project_id))
load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer')
secret_key = LocalState.get_secret_key(keyname)
admin_client = AdminClient(load_balancer_ip, secret_key)
operation_id = admin_client.update_dispatch(version.project_id, dispatch_rules)
# Check on the operation.
AppScaleLogger.log("Please wait for your dispatch to be updated.")
deadline = time.time() + cls.MAX_OPERATION_TIME
while True:
if time.time() > deadline:
raise AppScaleException('The operation took too long.')
operation = admin_client.get_operation(version.project_id, operation_id)
if not operation['done']:
time.sleep(1)
continue
if 'error' in operation:
raise AppScaleException(operation['error']['message'])
dispatch_rules = operation['response']['dispatchRules']
break
AppScaleLogger.verbose(
"The following dispatchRules have been applied to your application's "
"configuration : {}".format(dispatch_rules))
AppScaleLogger.success('Dispatch has been updated for {}'.format(
version.project_id))
def _remove_service(cls, admin_client, project_id, service_id):
""" Deletes a project's service.
Args:
admin_client: An AdminClient object.
project_id: A string specifying a project ID.
service_id: A string specifying a service ID.
Raises:
AppScaleException if the operation times out.
AdminError: If there is a problem making an Admin API call.
"""
operation_id = admin_client.delete_service(project_id, service_id)
deadline = time.time() + cls.MAX_OPERATION_TIME
while True:
if time.time() > deadline:
raise AppScaleException('The service delete operation timed out')
operation = admin_client.get_operation(project_id, operation_id)
if not operation['done']:
time.sleep(1)
continue
if 'error' in operation:
raise AppScaleException(operation['error']['message'])
break
def tail(self, node, file_regex):
""" 'tail' provides a simple way to follow log files in an AppScale
deployment, instead of having to ssh in to a machine, locate the logs
directory, and then tail it.
Args:
node: An int that indicates the id of the machine to tail logs from.
file_regex: The regular expression that should be used to indicate which
logs to tail from on the remote host.
Raises:
AppScalefileException: If there is no AppScalefile in the current working
directory.
TypeError: If index is not an int.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
# ensure that index is an int
# TODO(cgb): Consider node = *, to tail from all nodes.
try:
index = int(node)
except ValueError:
raise TypeError("Usage: appscale tail <node id to tail from> " + \
"<regex of files to tail>\nExample: appscale tail 0 controller*")
# get a list of the nodes running
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = "appscale"
try:
with open(self.get_locations_json_file(keyname)) as f:
nodes = json.loads(f.read()).get('node_info', [])
except IOError:
raise AppScaleException(
"AppScale does not currently appear to" +
" be running. Please start it and try again.")
# make sure there is a node at position 'index'
try:
ip = nodes[index]['public_ip']
except IndexError:
raise AppScaleException(
"Cannot tail from node at index " + str(index) +
", as there are only " + str(len(nodes)) +
" in the currently running AppScale deployment.")
# construct the ssh command to exec with that IP address
tail = "tail -F /var/log/appscale/{0}".format(file_regex)
command = [
"ssh", "-o", "StrictHostkeyChecking=no", "-i",
self.get_key_location(keyname), "root@" + ip, tail
]
# exec the ssh command
subprocess.call(command)
def ssh(self, node):
""" 'ssh' provides a simple way to log into virtual machines in an AppScale
deployment, using the SSH key provided in the user's AppScalefile.
Args:
node: An int that represents the node to SSH to. The value is used as an
index into the list of nodes running in the AppScale deployment,
starting with zero.
Raises:
AppScalefileException: If there is no AppScalefile in the current
directory.
TypeError: If the user does not provide an integer for 'node'.
"""
contents = self.read_appscalefile()
contents_as_yaml = yaml.safe_load(contents)
if 'keyname' in contents_as_yaml:
keyname = contents_as_yaml['keyname']
else:
keyname = "appscale"
if node is None:
node = "shadow"
try:
index = int(node)
nodes = self.get_nodes(keyname)
# make sure there is a node at position 'index'
ip = nodes[index]['public_ip']
except IndexError:
raise AppScaleException(
"Cannot ssh to node at index " + ", as there are only " +
str(len(nodes)) +
" in the currently running AppScale deployment.")
except ValueError:
try:
ip = LocalState.get_host_with_role(keyname, node.lower())
except AppScaleException:
raise AppScaleException("No role exists by that name. "
"Valid roles are {}".format(
NodeLayout.ADVANCED_FORMAT_KEYS))
# construct the ssh command to exec with that IP address
command = [
"ssh", "-o", "StrictHostkeyChecking=no", "-i",
self.get_key_location(keyname), "root@" + ip
]
# exec the ssh command
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
raise AppScaleException(
"Unable to ssh to the machine at "
"{}. Please make sure this machine is reachable, "
"has a public ip, or that the role is in use by "
"the deployment.".format(ip))
def stop_service(cls, options):
"""Instructs AppScale to stop the named service.
This is applicable for services using manual scaling.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If the named service isn't running in this
AppScale cloud, or if stop is not valid for the service.
"""
if not options.confirm:
response = raw_input(
'Are you sure you want to stop this service? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled service stop.")
load_balancer_ip = LocalState.get_host_with_role(
options.keyname, 'load_balancer')
secret = LocalState.get_secret_key(options.keyname)
admin_client = AdminClient(load_balancer_ip, secret)
version = Version(None, None)
version.project_id = options.project_id
version.service_id = options.service_id or DEFAULT_SERVICE
version.id = DEFAULT_VERSION
version.serving_status = 'STOPPED'
admin_client.patch_version(version, ['servingStatus'])
AppScaleLogger.success('Stop requested for {}.'.format(options.project_id))
def register(self, deployment_id):
""" Allows users to register their AppScale deployment with the AppScale
Portal.
Raises:
AppScaleException: If the deployment has already been registered.
"""
appscale_yaml = yaml.safe_load(self.read_appscalefile())
if 'keyname' in appscale_yaml:
keyname = appscale_yaml['keyname']
else:
keyname = 'appscale'
nodes = self.get_nodes(keyname)
head_node = self.get_head_node(nodes)
if RegistrationHelper.appscale_has_deployment_id(head_node, keyname):
existing_id = RegistrationHelper.get_deployment_id(head_node, keyname)
if existing_id != deployment_id:
raise AppScaleException(
'This deployment has already been registered with a different ID.')
if 'infrastructure' in appscale_yaml:
deployment_type = 'cloud'
else:
deployment_type = 'cluster'
deployment = RegistrationHelper.update_deployment(deployment_type, nodes,
deployment_id)
RegistrationHelper.set_deployment_id(head_node, keyname, deployment_id)
AppScaleLogger.success(
'Registration complete for AppScale deployment {0}.'
.format(deployment['name']))
def get_head_node(self, nodes):
""" Retrieve a node with the 'shadow' role.
Args:
nodes: A list of nodes in the running AppScale deployment.
Returns:
A string containing the IP address of the head node.
"""
for node in nodes:
if 'shadow' in node['jobs']:
return node['public_ip']
raise AppScaleException('Unable to find head node.')
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
private_key = path
else:
_, private_key = LocalState.generate_rsa_key(options.keyname)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password))
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip))
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def get_nodes(self, keyname):
""" Retrieve a list of the running nodes.
Args:
keyname: An identifier for the AppScale deployment.
Returns:
A list of nodes in the running AppScale deployment.
Raises:
AppScaleException: If there is no locations JSON file.
"""
try:
with open(self.get_locations_json_file(keyname)) as locations_file:
return json.loads(locations_file.read()).get('node_info', [])
except IOError:
raise AppScaleException("AppScale does not currently appear to"
" be running. Please start it and try again.")
def warn_if_version_defined(cls, version, test=False):
""" Warns the user if version is defined in the application configuration.
Args:
version: A Version object.
test: A boolean indicating that the tools are in test mode.
Raises:
AppScaleException: If version is defined and user decides to cancel.
"""
if version.id is not None:
AppScaleLogger.log(
'The version element is not supported in {}. Module {} will be '
'overwritten.'.format(version.config_type, version.service_id))
if not test:
response = raw_input('Continue? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException('Cancelled deploy operation')
def remove_service(cls, options):
"""Instructs AppScale to no longer host the named application.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if not options.confirm:
response = raw_input(
'Are you sure you want to remove this service? (y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled service removal.")
load_balancer_ip = LocalState.get_host_with_role(
options.keyname, 'load_balancer')
secret = LocalState.get_secret_key(options.keyname)
admin_client = AdminClient(load_balancer_ip, secret)
cls._remove_service(admin_client, options.project_id, options.service_id)
AppScaleLogger.success('Done shutting down service {} for {}.'.format(
options.project_id, options.service_id))
def relocate_app(cls, options):
"""Instructs AppScale to move the named application to a different port.
Args:
options: A Namespace that has fields for each parameter that can be passed
in via the command-line interface.
Raises:
AppScaleException: If the named application isn't running in this AppScale
cloud, if the destination port is in use by a different application, or
if the AppController rejects the request to relocate the application (in
which case it includes the reason why the rejection occurred).
"""
load_balancer_ip = LocalState.get_host_with_role(
options.keyname, 'load_balancer')
acc = AppControllerClient(
load_balancer_ip, LocalState.get_secret_key(options.keyname))
version_key = '_'.join([options.appname, DEFAULT_SERVICE, DEFAULT_VERSION])
app_info_map = acc.get_app_info_map()
if version_key not in app_info_map:
raise AppScaleException("The given application, {0}, is not currently " \
"running in this AppScale cloud, so we can't move it to a different " \
"port.".format(options.appname))
try:
login_host = acc.get_property('login')['login']
except KeyError:
raise AppControllerException('login property not found')
acc.relocate_version(version_key, options.http_port, options.https_port)
AppScaleLogger.success(
'Successfully issued request to move {0} to ports {1} and {2}'.format(
options.appname, options.http_port, options.https_port))
RemoteHelper.sleep_until_port_is_open(login_host, options.http_port)
AppScaleLogger.success(
'Your app serves unencrypted traffic at: http://{0}:{1}'.format(
login_host, options.http_port))
AppScaleLogger.success(
'Your app serves encrypted traffic at: https://{0}:{1}'.format(
login_host, options.https_port))
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
custom_service_yaml = None
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(options.file)
created_dir = True
version = Version.from_tar_gz(options.file)
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(options.file)
created_dir = True
version = Version.from_zip(options.file)
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
version = Version.from_directory(options.file)
elif options.file.endswith('.yaml'):
file_location = os.path.dirname(options.file)
created_dir = False
version = Version.from_yaml_file(options.file)
custom_service_yaml = options.file
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
if options.project:
if version.runtime == 'java':
raise BadConfigurationException("AppScale doesn't support --project for"
"Java yet. Please specify the application id in appengine-web.xml.")
version.project_id = options.project
if version.project_id is None:
if version.config_type == 'app.yaml':
message = 'Specify --project or define "application" in your app.yaml'
else:
message = 'Define "application" in your appengine-web.xml'
raise AppEngineConfigException(message)
# Let users know that versions are not supported yet.
AppEngineHelper.warn_if_version_defined(version, options.test)
AppEngineHelper.validate_app_id(version.project_id)
extras = {}
if version.runtime == 'go':
extras = LocalState.get_extra_go_dependencies(options.file, options.test)
if (version.runtime == 'java'
and AppEngineHelper.is_sdk_mismatch(file_location)):
AppScaleLogger.warn(
'AppScale did not find the correct SDK jar versions in your app. The '
'current supported SDK version is '
'{}.'.format(AppEngineHelper.SUPPORTED_SDK_VERSION))
head_node_public_ip = LocalState.get_host_with_role(
options.keyname, 'shadow')
secret_key = LocalState.get_secret_key(options.keyname)
admin_client = AdminClient(head_node_public_ip, secret_key)
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, version.project_id, options.keyname,
extras, custom_service_yaml)
AppScaleLogger.log(
'Deploying service {} for {}'.format(version.service_id,
version.project_id))
operation_id = admin_client.create_version(version, remote_file_path)
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
deadline = time.time() + cls.MAX_OPERATION_TIME
while True:
if time.time() > deadline:
raise AppScaleException('The deployment operation took too long.')
operation = admin_client.get_operation(version.project_id, operation_id)
if not operation['done']:
time.sleep(1)
continue
if 'error' in operation:
raise AppScaleException(operation['error']['message'])
version_url = operation['response']['versionUrl']
break
AppScaleLogger.success(
'Your app can be reached at the following URL: {}'.format(version_url))
if created_dir:
shutil.rmtree(file_location)
match = re.match('http://(.+):(\d+)', version_url)
login_host = match.group(1)
http_port = int(match.group(2))
return login_host, http_port
def terminate_instances(cls, options):
"""Stops all services running in an AppScale deployment, and in cloud
deployments, also powers off the instances previously spawned.
Raises:
AppScaleException: If AppScale is not running, and thus can't be
terminated.
"""
try:
infrastructure = LocalState.get_infrastructure(options.keyname)
except IOError:
raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".
format(options.keyname))
if infrastructure == "xen" and options.terminate:
raise AppScaleException("Terminate option is invalid for cluster mode.")
if infrastructure == "xen" or not options.terminate:
# We are in cluster mode: let's check if AppScale is running.
if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
raise AppScaleException("AppScale is not running with the keyname {0}".
format(options.keyname))
# Stop gracefully the AppScale deployment.
try:
RemoteHelper.terminate_virtualized_cluster(options.keyname,
options.clean)
except (IOError, AppScaleException, AppControllerException,
BadConfigurationException) as e:
if not (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
raise
if options.test:
AppScaleLogger.warn(e)
else:
AppScaleLogger.verbose(e)
if isinstance(e, AppControllerException):
response = raw_input(
'AppScale may not have shut down properly, are you sure you want '
'to continue terminating? (y/N) ')
else:
response = raw_input(
'AppScale could not find the configuration files for this '
'deployment, are you sure you want to continue terminating? '
'(y/N) ')
if response.lower() not in ['y', 'yes']:
raise AppScaleException("Cancelled cloud termination.")
# And if we are on a cloud infrastructure, terminate instances if
# asked.
if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and
options.terminate):
RemoteHelper.terminate_cloud_infrastructure(options.keyname)
elif infrastructure in InfrastructureAgentFactory.VALID_AGENTS and not \
options.terminate:
AppScaleLogger.log("AppScale did not terminate any of your cloud "
"instances, to terminate them run 'appscale "
"down --terminate'")
if options.clean:
LocalState.clean_local_metadata(keyname=options.keyname)
def gather_logs(cls, options):
"""Collects logs from each machine in the currently running AppScale
deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
location = os.path.abspath(options.location)
# First, make sure that the place we want to store logs doesn't
# already exist.
if os.path.exists(location):
raise AppScaleException("Can't gather logs, as the location you " + \
"specified, {}, already exists.".format(location))
load_balancer_ip = LocalState.get_host_with_role(
options.keyname, 'load_balancer')
secret = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(load_balancer_ip, secret)
try:
all_ips = acc.get_all_public_ips()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \
"machines in this AppScale deployment. Using our locally cached " + \
"info instead.")
all_ips = LocalState.get_all_public_ips(options.keyname)
# Get information about roles and public IPs
# for creating navigation symlinks in gathered logs
try:
nodes_info = acc.get_role_info()
except socket.error: # Occurs when the AppController has failed.
AppScaleLogger.warn("Couldn't get an up-to-date nodes info. "
"Using our locally cached info instead.")
nodes_info = LocalState.get_local_nodes_info(options.keyname)
nodes_dict = {node['public_ip']: node for node in nodes_info}
# do the mkdir after we get the secret key, so that a bad keyname will
# cause the tool to crash and not create this directory
os.mkdir(location)
# make dir for private IP navigation links
private_ips_dir = os.path.join(location, 'symlinks', 'private-ips')
utils.mkdir(private_ips_dir)
# The log paths that we collect logs from.
log_paths = [
{'remote': '/opt/cassandra/cassandra/logs/*', 'local': 'cassandra'},
{'remote': '/var/log/appscale'},
{'remote': '/var/log/haproxy.log*'},
{'remote': '/var/log/kern.log*'},
{'remote': '/var/log/nginx'},
{'remote': '/var/log/rabbitmq/*', 'local': 'rabbitmq'},
{'remote': '/var/log/syslog*'},
{'remote': '/var/log/zookeeper'}
]
failures = False
for public_ip in all_ips:
# Get the logs from each node, and store them in our local directory
local_dir = os.path.join(location, public_ip)
utils.mkdir(local_dir)
local_link = os.path.join('..', '..', public_ip)
# Create symlinks for easier navigation in gathered logs
node_info = nodes_dict.get(public_ip)
if node_info:
private_ip_dir = os.path.join(private_ips_dir, node_info["private_ip"])
os.symlink(local_link, private_ip_dir)
for role in node_info['roles']:
role_dir = os.path.join(location, 'symlinks', role)
utils.mkdir(role_dir)
os.symlink(local_link, os.path.join(role_dir, public_ip))
for log_path in log_paths:
sub_dir = local_dir
if 'local' in log_path:
sub_dir = os.path.join(local_dir, log_path['local'])
utils.mkdir(sub_dir)
try:
RemoteHelper.scp_remote_to_local(
public_ip, options.keyname, log_path['remote'], sub_dir
)
except ShellException as shell_exception:
failures = True
AppScaleLogger.warn('Unable to collect logs from {} for host {}'.
format(log_path['remote'], public_ip))
AppScaleLogger.verbose(
'Encountered exception: {}'.format(str(shell_exception)))
if failures:
AppScaleLogger.log("Done copying to {}. There were failures while "
"collecting AppScale logs.".format(location))
else:
AppScaleLogger.success("Successfully collected all AppScale logs into "
"{}".format(location))