def update_dispatch(cls, source_location, keyname, project_id): """ Updates an application's dispatch routing rules from the configuration file. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if cls.TAR_GZ_REGEX.search(source_location): fetch_function = utils.config_from_tar_gz version = Version.from_tar_gz(source_location) elif cls.ZIP_REGEX.search(source_location): fetch_function = utils.config_from_zip version = Version.from_zip(source_location) elif os.path.isdir(source_location): fetch_function = utils.config_from_dir version = Version.from_directory(source_location) elif source_location.endswith('.yaml'): fetch_function = utils.config_from_dir version = Version.from_yaml_file(source_location) source_location = os.path.dirname(source_location) else: raise BadConfigurationException( '{} must be a directory, tar.gz, or zip'.format(source_location)) if project_id: version.project_id = project_id dispatch_rules = utils.dispatch_from_yaml(source_location, fetch_function) if dispatch_rules is None: return AppScaleLogger.log('Updating dispatch for {}'.format(version.project_id)) load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret_key = LocalState.get_secret_key(keyname) admin_client = AdminClient(load_balancer_ip, secret_key) operation_id = admin_client.update_dispatch(version.project_id, dispatch_rules) # Check on the operation. AppScaleLogger.log("Please wait for your dispatch to be updated.") deadline = time.time() + cls.MAX_OPERATION_TIME while True: if time.time() > deadline: raise AppScaleException('The operation took too long.') operation = admin_client.get_operation(version.project_id, operation_id) if not operation['done']: time.sleep(1) continue if 'error' in operation: raise AppScaleException(operation['error']['message']) dispatch_rules = operation['response']['dispatchRules'] break AppScaleLogger.verbose( "The following dispatchRules have been applied to your application's " "configuration : {}".format(dispatch_rules)) AppScaleLogger.success('Dispatch has been updated for {}'.format( version.project_id))
def _remove_service(cls, admin_client, project_id, service_id): """ Deletes a project's service. Args: admin_client: An AdminClient object. project_id: A string specifying a project ID. service_id: A string specifying a service ID. Raises: AppScaleException if the operation times out. AdminError: If there is a problem making an Admin API call. """ operation_id = admin_client.delete_service(project_id, service_id) deadline = time.time() + cls.MAX_OPERATION_TIME while True: if time.time() > deadline: raise AppScaleException('The service delete operation timed out') operation = admin_client.get_operation(project_id, operation_id) if not operation['done']: time.sleep(1) continue if 'error' in operation: raise AppScaleException(operation['error']['message']) break
def tail(self, node, file_regex): """ 'tail' provides a simple way to follow log files in an AppScale deployment, instead of having to ssh in to a machine, locate the logs directory, and then tail it. Args: node: An int that indicates the id of the machine to tail logs from. file_regex: The regular expression that should be used to indicate which logs to tail from on the remote host. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. TypeError: If index is not an int. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # ensure that index is an int # TODO(cgb): Consider node = *, to tail from all nodes. try: index = int(node) except ValueError: raise TypeError("Usage: appscale tail <node id to tail from> " + \ "<regex of files to tail>\nExample: appscale tail 0 controller*") # get a list of the nodes running if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] else: keyname = "appscale" try: with open(self.get_locations_json_file(keyname)) as f: nodes = json.loads(f.read()).get('node_info', []) except IOError: raise AppScaleException( "AppScale does not currently appear to" + " be running. Please start it and try again.") # make sure there is a node at position 'index' try: ip = nodes[index]['public_ip'] except IndexError: raise AppScaleException( "Cannot tail from node at index " + str(index) + ", as there are only " + str(len(nodes)) + " in the currently running AppScale deployment.") # construct the ssh command to exec with that IP address tail = "tail -F /var/log/appscale/{0}".format(file_regex) command = [ "ssh", "-o", "StrictHostkeyChecking=no", "-i", self.get_key_location(keyname), "root@" + ip, tail ] # exec the ssh command subprocess.call(command)
def ssh(self, node): """ 'ssh' provides a simple way to log into virtual machines in an AppScale deployment, using the SSH key provided in the user's AppScalefile. Args: node: An int that represents the node to SSH to. The value is used as an index into the list of nodes running in the AppScale deployment, starting with zero. Raises: AppScalefileException: If there is no AppScalefile in the current directory. TypeError: If the user does not provide an integer for 'node'. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] else: keyname = "appscale" if node is None: node = "shadow" try: index = int(node) nodes = self.get_nodes(keyname) # make sure there is a node at position 'index' ip = nodes[index]['public_ip'] except IndexError: raise AppScaleException( "Cannot ssh to node at index " + ", as there are only " + str(len(nodes)) + " in the currently running AppScale deployment.") except ValueError: try: ip = LocalState.get_host_with_role(keyname, node.lower()) except AppScaleException: raise AppScaleException("No role exists by that name. " "Valid roles are {}".format( NodeLayout.ADVANCED_FORMAT_KEYS)) # construct the ssh command to exec with that IP address command = [ "ssh", "-o", "StrictHostkeyChecking=no", "-i", self.get_key_location(keyname), "root@" + ip ] # exec the ssh command try: subprocess.check_call(command) except subprocess.CalledProcessError: raise AppScaleException( "Unable to ssh to the machine at " "{}. Please make sure this machine is reachable, " "has a public ip, or that the role is in use by " "the deployment.".format(ip))
def stop_service(cls, options): """Instructs AppScale to stop the named service. This is applicable for services using manual scaling. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If the named service isn't running in this AppScale cloud, or if stop is not valid for the service. """ if not options.confirm: response = raw_input( 'Are you sure you want to stop this service? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled service stop.") load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(load_balancer_ip, secret) version = Version(None, None) version.project_id = options.project_id version.service_id = options.service_id or DEFAULT_SERVICE version.id = DEFAULT_VERSION version.serving_status = 'STOPPED' admin_client.patch_version(version, ['servingStatus']) AppScaleLogger.success('Stop requested for {}.'.format(options.project_id))
def register(self, deployment_id): """ Allows users to register their AppScale deployment with the AppScale Portal. Raises: AppScaleException: If the deployment has already been registered. """ appscale_yaml = yaml.safe_load(self.read_appscalefile()) if 'keyname' in appscale_yaml: keyname = appscale_yaml['keyname'] else: keyname = 'appscale' nodes = self.get_nodes(keyname) head_node = self.get_head_node(nodes) if RegistrationHelper.appscale_has_deployment_id(head_node, keyname): existing_id = RegistrationHelper.get_deployment_id(head_node, keyname) if existing_id != deployment_id: raise AppScaleException( 'This deployment has already been registered with a different ID.') if 'infrastructure' in appscale_yaml: deployment_type = 'cloud' else: deployment_type = 'cluster' deployment = RegistrationHelper.update_deployment(deployment_type, nodes, deployment_id) RegistrationHelper.set_deployment_id(head_node, keyname, deployment_id) AppScaleLogger.success( 'Registration complete for AppScale deployment {0}.' .format(deployment['name']))
def get_head_node(self, nodes): """ Retrieve a node with the 'shadow' role. Args: nodes: A list of nodes in the running AppScale deployment. Returns: A string containing the IP address of the head node. """ for node in nodes: if 'shadow' in node['jobs']: return node['public_ip'] raise AppScaleException('Unable to find head node.')
def add_keypair(cls, options): """Sets up passwordless SSH login to the machines used in a virtualized cluster deployment. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If any of the machines named in the ips_layout are not running, or do not have the SSH daemon running. """ LocalState.require_ssh_commands(options.auto) LocalState.make_appscale_directory() path = LocalState.LOCAL_APPSCALE_PATH + options.keyname if options.add_to_existing: private_key = path else: _, private_key = LocalState.generate_rsa_key(options.keyname) if options.auto: if 'root_password' in options: AppScaleLogger.log("Using the provided root password to log into " + \ "your VMs.") password = options.root_password else: AppScaleLogger.log("Please enter the password for the root user on" + \ " your VMs:") password = getpass.getpass() node_layout = NodeLayout(options) all_ips = [node.public_ip for node in node_layout.nodes] for ip in all_ips: # first, make sure ssh is actually running on the host machine if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT): raise AppScaleException("SSH does not appear to be running at {0}. " \ "Is the machine at {0} up and running? Make sure your IPs are " \ "correct!".format(ip)) # next, set up passwordless ssh AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip)) if options.auto: LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip, private_key, password)) else: LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip)) AppScaleLogger.success("Generated a new SSH key for this deployment " + \ "at {0}".format(private_key))
def get_nodes(self, keyname): """ Retrieve a list of the running nodes. Args: keyname: An identifier for the AppScale deployment. Returns: A list of nodes in the running AppScale deployment. Raises: AppScaleException: If there is no locations JSON file. """ try: with open(self.get_locations_json_file(keyname)) as locations_file: return json.loads(locations_file.read()).get('node_info', []) except IOError: raise AppScaleException("AppScale does not currently appear to" " be running. Please start it and try again.")
def warn_if_version_defined(cls, version, test=False): """ Warns the user if version is defined in the application configuration. Args: version: A Version object. test: A boolean indicating that the tools are in test mode. Raises: AppScaleException: If version is defined and user decides to cancel. """ if version.id is not None: AppScaleLogger.log( 'The version element is not supported in {}. Module {} will be ' 'overwritten.'.format(version.config_type, version.service_id)) if not test: response = raw_input('Continue? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException('Cancelled deploy operation')
def remove_service(cls, options): """Instructs AppScale to no longer host the named application. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if not options.confirm: response = raw_input( 'Are you sure you want to remove this service? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled service removal.") load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(load_balancer_ip, secret) cls._remove_service(admin_client, options.project_id, options.service_id) AppScaleLogger.success('Done shutting down service {} for {}.'.format( options.project_id, options.service_id))
def relocate_app(cls, options): """Instructs AppScale to move the named application to a different port. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If the named application isn't running in this AppScale cloud, if the destination port is in use by a different application, or if the AppController rejects the request to relocate the application (in which case it includes the reason why the rejection occurred). """ load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') acc = AppControllerClient( load_balancer_ip, LocalState.get_secret_key(options.keyname)) version_key = '_'.join([options.appname, DEFAULT_SERVICE, DEFAULT_VERSION]) app_info_map = acc.get_app_info_map() if version_key not in app_info_map: raise AppScaleException("The given application, {0}, is not currently " \ "running in this AppScale cloud, so we can't move it to a different " \ "port.".format(options.appname)) try: login_host = acc.get_property('login')['login'] except KeyError: raise AppControllerException('login property not found') acc.relocate_version(version_key, options.http_port, options.https_port) AppScaleLogger.success( 'Successfully issued request to move {0} to ports {1} and {2}'.format( options.appname, options.http_port, options.https_port)) RemoteHelper.sleep_until_port_is_open(login_host, options.http_port) AppScaleLogger.success( 'Your app serves unencrypted traffic at: http://{0}:{1}'.format( login_host, options.http_port)) AppScaleLogger.success( 'Your app serves encrypted traffic at: https://{0}:{1}'.format( login_host, options.https_port))
def upload_app(cls, options): """Uploads the given App Engine application into AppScale. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Returns: A tuple containing the host and port where the application is serving traffic from. """ custom_service_yaml = None if cls.TAR_GZ_REGEX.search(options.file): file_location = LocalState.extract_tgz_app_to_dir(options.file) created_dir = True version = Version.from_tar_gz(options.file) elif cls.ZIP_REGEX.search(options.file): file_location = LocalState.extract_zip_app_to_dir(options.file) created_dir = True version = Version.from_zip(options.file) elif os.path.isdir(options.file): file_location = options.file created_dir = False version = Version.from_directory(options.file) elif options.file.endswith('.yaml'): file_location = os.path.dirname(options.file) created_dir = False version = Version.from_yaml_file(options.file) custom_service_yaml = options.file else: raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \ 'or a directory. Please try uploading either a tar.gz file, a zip ' \ 'file, or a directory.'.format(options.file)) if options.project: if version.runtime == 'java': raise BadConfigurationException("AppScale doesn't support --project for" "Java yet. Please specify the application id in appengine-web.xml.") version.project_id = options.project if version.project_id is None: if version.config_type == 'app.yaml': message = 'Specify --project or define "application" in your app.yaml' else: message = 'Define "application" in your appengine-web.xml' raise AppEngineConfigException(message) # Let users know that versions are not supported yet. AppEngineHelper.warn_if_version_defined(version, options.test) AppEngineHelper.validate_app_id(version.project_id) extras = {} if version.runtime == 'go': extras = LocalState.get_extra_go_dependencies(options.file, options.test) if (version.runtime == 'java' and AppEngineHelper.is_sdk_mismatch(file_location)): AppScaleLogger.warn( 'AppScale did not find the correct SDK jar versions in your app. The ' 'current supported SDK version is ' '{}.'.format(AppEngineHelper.SUPPORTED_SDK_VERSION)) head_node_public_ip = LocalState.get_host_with_role( options.keyname, 'shadow') secret_key = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(head_node_public_ip, secret_key) remote_file_path = RemoteHelper.copy_app_to_host( file_location, version.project_id, options.keyname, extras, custom_service_yaml) AppScaleLogger.log( 'Deploying service {} for {}'.format(version.service_id, version.project_id)) operation_id = admin_client.create_version(version, remote_file_path) # now that we've told the AppController to start our app, find out what port # the app is running on and wait for it to start serving AppScaleLogger.log("Please wait for your app to start serving.") deadline = time.time() + cls.MAX_OPERATION_TIME while True: if time.time() > deadline: raise AppScaleException('The deployment operation took too long.') operation = admin_client.get_operation(version.project_id, operation_id) if not operation['done']: time.sleep(1) continue if 'error' in operation: raise AppScaleException(operation['error']['message']) version_url = operation['response']['versionUrl'] break AppScaleLogger.success( 'Your app can be reached at the following URL: {}'.format(version_url)) if created_dir: shutil.rmtree(file_location) match = re.match('http://(.+):(\d+)', version_url) login_host = match.group(1) http_port = int(match.group(2)) return login_host, http_port
def terminate_instances(cls, options): """Stops all services running in an AppScale deployment, and in cloud deployments, also powers off the instances previously spawned. Raises: AppScaleException: If AppScale is not running, and thus can't be terminated. """ try: infrastructure = LocalState.get_infrastructure(options.keyname) except IOError: raise AppScaleException("Cannot find AppScale's configuration for keyname {0}". format(options.keyname)) if infrastructure == "xen" and options.terminate: raise AppScaleException("Terminate option is invalid for cluster mode.") if infrastructure == "xen" or not options.terminate: # We are in cluster mode: let's check if AppScale is running. if not os.path.exists(LocalState.get_secret_key_location(options.keyname)): raise AppScaleException("AppScale is not running with the keyname {0}". format(options.keyname)) # Stop gracefully the AppScale deployment. try: RemoteHelper.terminate_virtualized_cluster(options.keyname, options.clean) except (IOError, AppScaleException, AppControllerException, BadConfigurationException) as e: if not (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate): raise if options.test: AppScaleLogger.warn(e) else: AppScaleLogger.verbose(e) if isinstance(e, AppControllerException): response = raw_input( 'AppScale may not have shut down properly, are you sure you want ' 'to continue terminating? (y/N) ') else: response = raw_input( 'AppScale could not find the configuration files for this ' 'deployment, are you sure you want to continue terminating? ' '(y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled cloud termination.") # And if we are on a cloud infrastructure, terminate instances if # asked. if (infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate): RemoteHelper.terminate_cloud_infrastructure(options.keyname) elif infrastructure in InfrastructureAgentFactory.VALID_AGENTS and not \ options.terminate: AppScaleLogger.log("AppScale did not terminate any of your cloud " "instances, to terminate them run 'appscale " "down --terminate'") if options.clean: LocalState.clean_local_metadata(keyname=options.keyname)
def gather_logs(cls, options): """Collects logs from each machine in the currently running AppScale deployment. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ location = os.path.abspath(options.location) # First, make sure that the place we want to store logs doesn't # already exist. if os.path.exists(location): raise AppScaleException("Can't gather logs, as the location you " + \ "specified, {}, already exists.".format(location)) load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) acc = AppControllerClient(load_balancer_ip, secret) try: all_ips = acc.get_all_public_ips() except socket.error: # Occurs when the AppController has failed. AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \ "machines in this AppScale deployment. Using our locally cached " + \ "info instead.") all_ips = LocalState.get_all_public_ips(options.keyname) # Get information about roles and public IPs # for creating navigation symlinks in gathered logs try: nodes_info = acc.get_role_info() except socket.error: # Occurs when the AppController has failed. AppScaleLogger.warn("Couldn't get an up-to-date nodes info. " "Using our locally cached info instead.") nodes_info = LocalState.get_local_nodes_info(options.keyname) nodes_dict = {node['public_ip']: node for node in nodes_info} # do the mkdir after we get the secret key, so that a bad keyname will # cause the tool to crash and not create this directory os.mkdir(location) # make dir for private IP navigation links private_ips_dir = os.path.join(location, 'symlinks', 'private-ips') utils.mkdir(private_ips_dir) # The log paths that we collect logs from. log_paths = [ {'remote': '/opt/cassandra/cassandra/logs/*', 'local': 'cassandra'}, {'remote': '/var/log/appscale'}, {'remote': '/var/log/haproxy.log*'}, {'remote': '/var/log/kern.log*'}, {'remote': '/var/log/nginx'}, {'remote': '/var/log/rabbitmq/*', 'local': 'rabbitmq'}, {'remote': '/var/log/syslog*'}, {'remote': '/var/log/zookeeper'} ] failures = False for public_ip in all_ips: # Get the logs from each node, and store them in our local directory local_dir = os.path.join(location, public_ip) utils.mkdir(local_dir) local_link = os.path.join('..', '..', public_ip) # Create symlinks for easier navigation in gathered logs node_info = nodes_dict.get(public_ip) if node_info: private_ip_dir = os.path.join(private_ips_dir, node_info["private_ip"]) os.symlink(local_link, private_ip_dir) for role in node_info['roles']: role_dir = os.path.join(location, 'symlinks', role) utils.mkdir(role_dir) os.symlink(local_link, os.path.join(role_dir, public_ip)) for log_path in log_paths: sub_dir = local_dir if 'local' in log_path: sub_dir = os.path.join(local_dir, log_path['local']) utils.mkdir(sub_dir) try: RemoteHelper.scp_remote_to_local( public_ip, options.keyname, log_path['remote'], sub_dir ) except ShellException as shell_exception: failures = True AppScaleLogger.warn('Unable to collect logs from {} for host {}'. format(log_path['remote'], public_ip)) AppScaleLogger.verbose( 'Encountered exception: {}'.format(str(shell_exception))) if failures: AppScaleLogger.log("Done copying to {}. There were failures while " "collecting AppScale logs.".format(location)) else: AppScaleLogger.success("Successfully collected all AppScale logs into " "{}".format(location))