def start_service(cls, options): """Instructs AppScale to start the named service. This is applicable for services using manual scaling. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If the named service isn't running in this AppScale cloud, or if start is not valid for the service. """ load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(load_balancer_ip, secret) version = Version(None, None) version.project_id = options.project_id version.service_id = options.service_id or DEFAULT_SERVICE version.id = DEFAULT_VERSION version.serving_status = 'SERVING' admin_client.patch_version(version, ['servingStatus']) AppScaleLogger.success('Start requested for {}.'.format(options.project_id))
def _get_stats(keyname, stats_kind, include_lists): """ Returns statistics from Hermes. Args: keyname: A string representing an identifier from AppScaleFile. stats_kind: A string representing a kind of statistics. include_lists: A dict representing desired fields. Returns: A dict of statistics. A dict of failures. """ load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret = LocalState.get_secret_key(keyname=keyname) administration_port = "17441" stats_path = "/stats/cluster/{stats_kind}".format(stats_kind=stats_kind) headers = {'Appscale-Secret': secret} data = {'include_lists': include_lists} url = "https://{ip}:{port}{path}".format(ip=load_balancer_ip, port=administration_port, path=stats_path) try: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) resp = requests.get(url=url, headers=headers, json=data, verify=False) resp.raise_for_status() except requests.HTTPError as err: AppScaleLogger.warn("Failed to get {stats_kind} stats ({err})".format( stats_kind=stats_kind, err=err)) return {}, {} json_body = resp.json() return json_body["stats"], json_body["failures"]
def stop_service(cls, options): """Instructs AppScale to stop the named service. This is applicable for services using manual scaling. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If the named service isn't running in this AppScale cloud, or if stop is not valid for the service. """ if not options.confirm: response = raw_input( 'Are you sure you want to stop this service? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled service stop.") load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(load_balancer_ip, secret) version = Version(None, None) version.project_id = options.project_id version.service_id = options.service_id or DEFAULT_SERVICE version.id = DEFAULT_VERSION version.serving_status = 'STOPPED' admin_client.patch_version(version, ['servingStatus']) AppScaleLogger.success('Stop requested for {}.'.format(options.project_id))
def update_dispatch(cls, source_location, keyname, project_id): """ Updates an application's dispatch routing rules from the configuration file. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if cls.TAR_GZ_REGEX.search(source_location): fetch_function = utils.config_from_tar_gz version = Version.from_tar_gz(source_location) elif cls.ZIP_REGEX.search(source_location): fetch_function = utils.config_from_zip version = Version.from_zip(source_location) elif os.path.isdir(source_location): fetch_function = utils.config_from_dir version = Version.from_directory(source_location) elif source_location.endswith('.yaml'): fetch_function = utils.config_from_dir version = Version.from_yaml_file(source_location) source_location = os.path.dirname(source_location) else: raise BadConfigurationException( '{} must be a directory, tar.gz, or zip'.format(source_location)) if project_id: version.project_id = project_id dispatch_rules = utils.dispatch_from_yaml(source_location, fetch_function) if dispatch_rules is None: return AppScaleLogger.log('Updating dispatch for {}'.format(version.project_id)) load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret_key = LocalState.get_secret_key(keyname) admin_client = AdminClient(load_balancer_ip, secret_key) operation_id = admin_client.update_dispatch(version.project_id, dispatch_rules) # Check on the operation. AppScaleLogger.log("Please wait for your dispatch to be updated.") deadline = time.time() + cls.MAX_OPERATION_TIME while True: if time.time() > deadline: raise AppScaleException('The operation took too long.') operation = admin_client.get_operation(version.project_id, operation_id) if not operation['done']: time.sleep(1) continue if 'error' in operation: raise AppScaleException(operation['error']['message']) dispatch_rules = operation['response']['dispatchRules'] break AppScaleLogger.verbose( "The following dispatchRules have been applied to your application's " "configuration : {}".format(dispatch_rules)) AppScaleLogger.success('Dispatch has been updated for {}'.format( version.project_id))
def create_user(cls, options, is_admin): """Create a new user with the parameters given. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. is_admin: A flag to indicate if the user to be created is an admin user Raises: AppControllerException: If the AppController on the head node crashes. When this occurs, the message in the exception contains the reason why the AppController crashed. """ secret = LocalState.get_secret_key(options.keyname) load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') username, password = LocalState.get_credentials(is_admin) acc = AppControllerClient(load_balancer_ip, secret) RemoteHelper.create_user_accounts( username, password, load_balancer_ip, options.keyname) try: if is_admin: acc.set_admin_role(username, 'true', cls.ADMIN_CAPABILITIES) except Exception as exception: AppScaleLogger.warn("Could not grant admin privileges to the user for the " + "following reason: {0}".format(str(exception))) sys.exit(1)
def add_instances(cls, options): """Adds additional machines to an AppScale deployment. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if 'master' in options.ips.keys(): raise BadConfigurationException("Cannot add master nodes to an " + \ "already running AppScale deployment.") # In virtualized cluster deployments, we need to make sure that the user # has already set up SSH keys. if LocalState.get_infrastructure_option(keyname=options.keyname, tag='infrastructure') == "xen": ips_to_check = [] for ip_group in options.ips.values(): ips_to_check.extend(ip_group) for ip in ips_to_check: # throws a ShellException if the SSH key doesn't work RemoteHelper.ssh(ip, options.keyname, "ls") # Finally, find an AppController and send it a message to add # the given nodes with the new roles. AppScaleLogger.log("Sending request to add instances") load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') acc = AppControllerClient(load_balancer_ip, LocalState.get_secret_key( options.keyname)) acc.start_roles_on_nodes(json.dumps(options.ips)) # TODO(cgb): Should we wait for the new instances to come up and get # initialized? AppScaleLogger.success("Successfully sent request to add instances " + \ "to this AppScale deployment.")
def print_cluster_status(cls, options): """ Gets cluster stats and prints it nicely. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ try: load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') acc = AppControllerClient( load_balancer_ip, LocalState.get_secret_key(options.keyname)) all_private_ips = acc.get_all_private_ips() cluster_stats = acc.get_cluster_stats() except (faultType, AppControllerException, BadConfigurationException): AppScaleLogger.warn("AppScale deployment is probably down") raise # Convert cluster stats to useful structures node_stats = { ip: next((n for n in cluster_stats if n["private_ip"] == ip), None) for ip in all_private_ips } apps_dict = next((n["apps"] for n in cluster_stats if n["apps"]), {}) services = [ServiceInfo(key.split('_')[0], key.split('_')[1], app_info) for key, app_info in apps_dict.iteritems()] nodes = [NodeStats(ip, node) for ip, node in node_stats.iteritems() if node] invisible_nodes = [ip for ip, node in node_stats.iteritems() if not node] if options.verbose: AppScaleLogger.log("-"*76) cls._print_nodes_info(nodes, invisible_nodes) cls._print_roles_info(nodes) else: AppScaleLogger.log("-"*76) cls._print_cluster_summary(nodes, invisible_nodes, services) cls._print_services(services) cls._print_status_alerts(nodes) try: login_host = acc.get_property('login')['login'] except KeyError: raise AppControllerException('login property not found') dashboard = next( (service for service in services if service.http == RemoteHelper.APP_DASHBOARD_PORT), None) if dashboard and dashboard.appservers >= 1: AppScaleLogger.success( "\nView more about your AppScale deployment at http://{}:{}/status" .format(login_host, RemoteHelper.APP_DASHBOARD_PORT) ) else: AppScaleLogger.log( "\nAs soon as AppScale Dashboard is started you can visit it at " "http://{0}:{1}/status and see more about your deployment" .format(login_host, RemoteHelper.APP_DASHBOARD_PORT) )
def ssh(self, node): """ 'ssh' provides a simple way to log into virtual machines in an AppScale deployment, using the SSH key provided in the user's AppScalefile. Args: node: An int that represents the node to SSH to. The value is used as an index into the list of nodes running in the AppScale deployment, starting with zero. Raises: AppScalefileException: If there is no AppScalefile in the current directory. TypeError: If the user does not provide an integer for 'node'. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] else: keyname = "appscale" if node is None: node = "shadow" try: index = int(node) nodes = self.get_nodes(keyname) # make sure there is a node at position 'index' ip = nodes[index]['public_ip'] except IndexError: raise AppScaleException( "Cannot ssh to node at index " + ", as there are only " + str(len(nodes)) + " in the currently running AppScale deployment.") except ValueError: try: ip = LocalState.get_host_with_role(keyname, node.lower()) except AppScaleException: raise AppScaleException("No role exists by that name. " "Valid roles are {}".format( NodeLayout.ADVANCED_FORMAT_KEYS)) # construct the ssh command to exec with that IP address command = [ "ssh", "-o", "StrictHostkeyChecking=no", "-i", self.get_key_location(keyname), "root@" + ip ] # exec the ssh command try: subprocess.check_call(command) except subprocess.CalledProcessError: raise AppScaleException( "Unable to ssh to the machine at " "{}. Please make sure this machine is reachable, " "has a public ip, or that the role is in use by " "the deployment.".format(ip))
def ssh(self, node): """ 'ssh' provides a simple way to log into virtual machines in an AppScale deployment, using the SSH key provided in the user's AppScalefile. Args: node: An int that represents the node to SSH to. The value is used as an index into the list of nodes running in the AppScale deployment, starting with zero. Raises: AppScalefileException: If there is no AppScalefile in the current directory. TypeError: If the user does not provide an integer for 'node'. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) if 'keyname' in contents_as_yaml: keyname = contents_as_yaml['keyname'] else: keyname = "appscale" if node is None: node = "shadow" try: index = int(node) nodes = self.get_nodes(keyname) # make sure there is a node at position 'index' ip = nodes[index]['public_ip'] except IndexError: raise AppScaleException("Cannot ssh to node at index " + ", as there are only " + str(len(nodes)) + " in the currently running AppScale deployment.") except ValueError: try: ip = LocalState.get_host_with_role(keyname, node.lower()) except AppScaleException: raise AppScaleException("No role exists by that name. " "Valid roles are {}" .format(NodeLayout.ADVANCED_FORMAT_KEYS)) # construct the ssh command to exec with that IP address command = ["ssh", "-o", "StrictHostkeyChecking=no", "-i", self.get_key_location(keyname), "root@" + ip] # exec the ssh command try: subprocess.check_call(command) except subprocess.CalledProcessError: raise AppScaleException("Unable to ssh to the machine at " "{}. Please make sure this machine is reachable, " "has a public ip, or that the role is in use by " "the deployment.".format(ip))
def set_property(cls, options): """Instructs AppScale to replace the value it uses for a particular AppController instance variable (property) with a new value. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow') acc = AppControllerClient(shadow_host, LocalState.get_secret_key( options.keyname)) acc.set_property(options.property_name, options.property_value) AppScaleLogger.success('Successfully updated the given property.')
def update_queues(cls, source_location, keyname, project_id): """ Updates a project's queues from the configuration file. Args: source_location: A string specifying the location of the source code. keyname: A string specifying the key name. project_id: A string specifying the project ID. """ if cls.TAR_GZ_REGEX.search(source_location): fetch_function = utils.config_from_tar_gz version = Version.from_tar_gz(source_location) elif cls.ZIP_REGEX.search(source_location): fetch_function = utils.config_from_zip version = Version.from_zip(source_location) elif os.path.isdir(source_location): fetch_function = utils.config_from_dir version = Version.from_directory(source_location) elif source_location.endswith('.yaml'): fetch_function = utils.config_from_dir version = Version.from_yaml_file(source_location) source_location = os.path.dirname(source_location) else: raise BadConfigurationException( '{} must be a directory, tar.gz, or zip'.format(source_location)) if project_id: version.project_id = project_id queue_config = fetch_function('queue.yaml', source_location) if queue_config is None: queue_config = fetch_function('queue.xml', source_location) # If the source does not have a queue configuration file, do nothing. if queue_config is None: return queues = utils.queues_from_xml(queue_config) else: queues = yaml.safe_load(queue_config) AppScaleLogger.log('Updating queues') for queue in queues.get('queue', []): if 'bucket_size' in queue or 'max_concurrent_requests' in queue: AppScaleLogger.warn('Queue configuration uses unsupported rate options' ' (bucket size or max concurrent requests)') break load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret_key = LocalState.get_secret_key(keyname) admin_client = AdminClient(load_balancer_ip, secret_key) admin_client.update_queues(version.project_id, queues)
def get_property(cls, options): """Queries AppScale for a list of system properties matching the provided regular expression, as well as the values associated with each matching property. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Returns: A dict mapping each property matching the given regex to its associated value. """ shadow_host = LocalState.get_host_with_role(options.keyname, 'shadow') acc = AppControllerClient(shadow_host, LocalState.get_secret_key( options.keyname)) return acc.get_property(options.property)
def _get_stats(keyname, stats_kind, include_lists): """ Returns statistics from Hermes. Args: keyname: A string representing an identifier from AppScaleFile. stats_kind: A string representing a kind of statistics. include_lists: A dict representing desired fields. Returns: A dict of statistics. A dict of failures. """ load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret = LocalState.get_secret_key(keyname=keyname) administration_port = "17441" stats_path = "/stats/cluster/{stats_kind}".format(stats_kind=stats_kind) headers = {'Appscale-Secret': secret} data = {'include_lists': include_lists} url = "https://{ip}:{port}{path}".format( ip=load_balancer_ip, port=administration_port, path=stats_path ) try: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) resp = requests.get( url=url, headers=headers, json=data, verify=False ) resp.raise_for_status() except requests.HTTPError as err: AppScaleLogger.warn( "Failed to get {stats_kind} stats ({err})" .format(stats_kind=stats_kind, err=err) ) return {}, {} json_body = resp.json() return json_body["stats"], json_body["failures"]
def update_cron(cls, source_location, keyname, project_id): """ Updates a project's cron jobs from the configuration file. Args: source_location: A string specifying the location of the source code. keyname: A string specifying the key name. project_id: A string specifying the project ID. """ if cls.TAR_GZ_REGEX.search(source_location): fetch_function = utils.config_from_tar_gz version = Version.from_tar_gz(source_location) elif cls.ZIP_REGEX.search(source_location): fetch_function = utils.config_from_zip version = Version.from_zip(source_location) elif os.path.isdir(source_location): fetch_function = utils.config_from_dir version = Version.from_directory(source_location) elif source_location.endswith('.yaml'): fetch_function = utils.config_from_dir version = Version.from_yaml_file(source_location) source_location = os.path.dirname(source_location) else: raise BadConfigurationException( '{} must be a directory, tar.gz, or zip'.format(source_location)) if project_id: version.project_id = project_id cron_config = fetch_function('cron.yaml', source_location) if cron_config is None: cron_config = fetch_function('cron.xml', source_location) # If the source does not have a cron configuration file, do nothing. if cron_config is None: return cron_jobs = utils.cron_from_xml(cron_config) else: cron_jobs = yaml.safe_load(cron_config) AppScaleLogger.log('Updating cron jobs') load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') secret_key = LocalState.get_secret_key(keyname) admin_client = AdminClient(load_balancer_ip, secret_key) admin_client.update_cron(version.project_id, cron_jobs)
def remove_service(cls, options): """Instructs AppScale to no longer host the named application. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ if not options.confirm: response = raw_input( 'Are you sure you want to remove this service? (y/N) ') if response.lower() not in ['y', 'yes']: raise AppScaleException("Cancelled service removal.") load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(load_balancer_ip, secret) cls._remove_service(admin_client, options.project_id, options.service_id) AppScaleLogger.success('Done shutting down service {} for {}.'.format( options.project_id, options.service_id))
def relocate_app(cls, options): """Instructs AppScale to move the named application to a different port. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Raises: AppScaleException: If the named application isn't running in this AppScale cloud, if the destination port is in use by a different application, or if the AppController rejects the request to relocate the application (in which case it includes the reason why the rejection occurred). """ load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') acc = AppControllerClient( load_balancer_ip, LocalState.get_secret_key(options.keyname)) version_key = '_'.join([options.appname, DEFAULT_SERVICE, DEFAULT_VERSION]) app_info_map = acc.get_app_info_map() if version_key not in app_info_map: raise AppScaleException("The given application, {0}, is not currently " \ "running in this AppScale cloud, so we can't move it to a different " \ "port.".format(options.appname)) try: login_host = acc.get_property('login')['login'] except KeyError: raise AppControllerException('login property not found') acc.relocate_version(version_key, options.http_port, options.https_port) AppScaleLogger.success( 'Successfully issued request to move {0} to ports {1} and {2}'.format( options.appname, options.http_port, options.https_port)) RemoteHelper.sleep_until_port_is_open(login_host, options.http_port) AppScaleLogger.success( 'Your app serves unencrypted traffic at: http://{0}:{1}'.format( login_host, options.http_port)) AppScaleLogger.success( 'Your app serves encrypted traffic at: https://{0}:{1}'.format( login_host, options.https_port))
def get_roles(keyname): """ Obtains roles for each ip from AppControllerClient. Args: keyname: A string representing an identifier from AppScaleFile. Returns: A dict in which each key is an ip and value is a role list. """ load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') acc = AppControllerClient(host=load_balancer_ip, secret=LocalState.get_secret_key(keyname)) cluster_stats = acc.get_cluster_stats() roles_data = { node["private_ip"]: (node["roles"] if len(node["roles"]) > 0 else ["?"]) for node in cluster_stats } return roles_data
def get_roles(keyname): """ Obtains roles for each ip from AppControllerClient. Args: keyname: A string representing an identifier from AppScaleFile. Returns: A dict in which each key is an ip and value is a role list. """ load_balancer_ip = LocalState.get_host_with_role(keyname, 'load_balancer') acc = AppControllerClient( host=load_balancer_ip, secret=LocalState.get_secret_key(keyname) ) cluster_stats = acc.get_cluster_stats() roles_data = { node["private_ip"]: (node["roles"] if len(node["roles"]) > 0 else ["?"]) for node in cluster_stats } return roles_data
def reset_password(cls, options): """Resets a user's password the currently running AppScale deployment. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ secret = LocalState.get_secret_key(options.keyname) load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') username, password = LocalState.get_credentials(is_admin=False) encrypted_password = LocalState.encrypt_password(username, password) acc = AppControllerClient(load_balancer_ip, secret) try: acc.reset_password(username, encrypted_password) AppScaleLogger.success("The password was successfully changed for the " \ "given user.") except Exception as exception: AppScaleLogger.warn("Could not change the user's password for the " + \ "following reason: {0}".format(str(exception))) sys.exit(1)
def upload_app(cls, options): """Uploads the given App Engine application into AppScale. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. Returns: A tuple containing the host and port where the application is serving traffic from. """ custom_service_yaml = None if cls.TAR_GZ_REGEX.search(options.file): file_location = LocalState.extract_tgz_app_to_dir(options.file) created_dir = True version = Version.from_tar_gz(options.file) elif cls.ZIP_REGEX.search(options.file): file_location = LocalState.extract_zip_app_to_dir(options.file) created_dir = True version = Version.from_zip(options.file) elif os.path.isdir(options.file): file_location = options.file created_dir = False version = Version.from_directory(options.file) elif options.file.endswith('.yaml'): file_location = os.path.dirname(options.file) created_dir = False version = Version.from_yaml_file(options.file) custom_service_yaml = options.file else: raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \ 'or a directory. Please try uploading either a tar.gz file, a zip ' \ 'file, or a directory.'.format(options.file)) if options.project: if version.runtime == 'java': raise BadConfigurationException("AppScale doesn't support --project for" "Java yet. Please specify the application id in appengine-web.xml.") version.project_id = options.project if version.project_id is None: if version.config_type == 'app.yaml': message = 'Specify --project or define "application" in your app.yaml' else: message = 'Define "application" in your appengine-web.xml' raise AppEngineConfigException(message) # Let users know that versions are not supported yet. AppEngineHelper.warn_if_version_defined(version, options.test) AppEngineHelper.validate_app_id(version.project_id) extras = {} if version.runtime == 'go': extras = LocalState.get_extra_go_dependencies(options.file, options.test) if (version.runtime == 'java' and AppEngineHelper.is_sdk_mismatch(file_location)): AppScaleLogger.warn( 'AppScale did not find the correct SDK jar versions in your app. The ' 'current supported SDK version is ' '{}.'.format(AppEngineHelper.SUPPORTED_SDK_VERSION)) head_node_public_ip = LocalState.get_host_with_role( options.keyname, 'shadow') secret_key = LocalState.get_secret_key(options.keyname) admin_client = AdminClient(head_node_public_ip, secret_key) remote_file_path = RemoteHelper.copy_app_to_host( file_location, version.project_id, options.keyname, extras, custom_service_yaml) AppScaleLogger.log( 'Deploying service {} for {}'.format(version.service_id, version.project_id)) operation_id = admin_client.create_version(version, remote_file_path) # now that we've told the AppController to start our app, find out what port # the app is running on and wait for it to start serving AppScaleLogger.log("Please wait for your app to start serving.") deadline = time.time() + cls.MAX_OPERATION_TIME while True: if time.time() > deadline: raise AppScaleException('The deployment operation took too long.') operation = admin_client.get_operation(version.project_id, operation_id) if not operation['done']: time.sleep(1) continue if 'error' in operation: raise AppScaleException(operation['error']['message']) version_url = operation['response']['versionUrl'] break AppScaleLogger.success( 'Your app can be reached at the following URL: {}'.format(version_url)) if created_dir: shutil.rmtree(file_location) match = re.match('http://(.+):(\d+)', version_url) login_host = match.group(1) http_port = int(match.group(2)) return login_host, http_port
def gather_logs(cls, options): """Collects logs from each machine in the currently running AppScale deployment. Args: options: A Namespace that has fields for each parameter that can be passed in via the command-line interface. """ location = os.path.abspath(options.location) # First, make sure that the place we want to store logs doesn't # already exist. if os.path.exists(location): raise AppScaleException("Can't gather logs, as the location you " + \ "specified, {}, already exists.".format(location)) load_balancer_ip = LocalState.get_host_with_role( options.keyname, 'load_balancer') secret = LocalState.get_secret_key(options.keyname) acc = AppControllerClient(load_balancer_ip, secret) try: all_ips = acc.get_all_public_ips() except socket.error: # Occurs when the AppController has failed. AppScaleLogger.warn("Couldn't get an up-to-date listing of the " + \ "machines in this AppScale deployment. Using our locally cached " + \ "info instead.") all_ips = LocalState.get_all_public_ips(options.keyname) # Get information about roles and public IPs # for creating navigation symlinks in gathered logs try: nodes_info = acc.get_role_info() except socket.error: # Occurs when the AppController has failed. AppScaleLogger.warn("Couldn't get an up-to-date nodes info. " "Using our locally cached info instead.") nodes_info = LocalState.get_local_nodes_info(options.keyname) nodes_dict = {node['public_ip']: node for node in nodes_info} # do the mkdir after we get the secret key, so that a bad keyname will # cause the tool to crash and not create this directory os.mkdir(location) # make dir for private IP navigation links private_ips_dir = os.path.join(location, 'symlinks', 'private-ips') utils.mkdir(private_ips_dir) # The log paths that we collect logs from. log_paths = [ {'remote': '/opt/cassandra/cassandra/logs/*', 'local': 'cassandra'}, {'remote': '/var/log/appscale'}, {'remote': '/var/log/haproxy.log*'}, {'remote': '/var/log/kern.log*'}, {'remote': '/var/log/nginx'}, {'remote': '/var/log/rabbitmq/*', 'local': 'rabbitmq'}, {'remote': '/var/log/syslog*'}, {'remote': '/var/log/zookeeper'} ] failures = False for public_ip in all_ips: # Get the logs from each node, and store them in our local directory local_dir = os.path.join(location, public_ip) utils.mkdir(local_dir) local_link = os.path.join('..', '..', public_ip) # Create symlinks for easier navigation in gathered logs node_info = nodes_dict.get(public_ip) if node_info: private_ip_dir = os.path.join(private_ips_dir, node_info["private_ip"]) os.symlink(local_link, private_ip_dir) for role in node_info['roles']: role_dir = os.path.join(location, 'symlinks', role) utils.mkdir(role_dir) os.symlink(local_link, os.path.join(role_dir, public_ip)) for log_path in log_paths: sub_dir = local_dir if 'local' in log_path: sub_dir = os.path.join(local_dir, log_path['local']) utils.mkdir(sub_dir) try: RemoteHelper.scp_remote_to_local( public_ip, options.keyname, log_path['remote'], sub_dir ) except ShellException as shell_exception: failures = True AppScaleLogger.warn('Unable to collect logs from {} for host {}'. format(log_path['remote'], public_ip)) AppScaleLogger.verbose( 'Encountered exception: {}'.format(str(shell_exception))) if failures: AppScaleLogger.log("Done copying to {}. There were failures while " "collecting AppScale logs.".format(location)) else: AppScaleLogger.success("Successfully collected all AppScale logs into " "{}".format(location))