def hint(self): """ Provides a hint about finding the root cause of this recipe """ print('Giving hint for recipe') Recipe._auth_cluster() external_ip_command = "kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'" ip, error = Recipe._run_command(external_ip_command) ip = ip.decode("utf-8").replace("'", '') if not ip: print('No external IP found.') logging.error('No external IP found.') exit(1) print( 'Visit the external IP of the demo application to see if there are any visible changes: http://{}' .format(ip)) get_project_command = "gcloud config list --format value(core.project)" project_id, error = Recipe._run_command(get_project_command) project_id = project_id.decode("utf-8").replace('"', '') if not project_id: print('No project ID found.') logging.error('No project ID found.') exit(1) print( 'Use Monitoring Dashboards to see metrics associated with each service: https://console.cloud.google.com/monitoring/dashboards?project={}' .format(project_id)) print( 'Note: It may take up to 5 minutes for monitoring metrics to be updated' )
def hint(self): """ Provides a hint about the root cause of the issue """ get_project_command = "gcloud config list --format value(core.project)" project_id, error = Recipe._run_command(get_project_command) project_id = project_id.decode("utf-8").replace('"', '') if not project_id: print('No project ID found.') logging.error('No project ID found.') exit(1) print( 'Use Cloud Logging to view logs exported by each service: https://console.cloud.google.com/logs?project={}' .format(project_id))
def restore_service(self): """ Resume Cloud scheduler job to restore calls to recollect API of the rating service """ logging.info("Resuming scheduled job of the rating service") print("Restoring broken operations...") project_id = Recipe._get_project_id() if not project_id: print("Failed: cannot find project id.") logging.error("Failed pausing scheduled job: no project id.") exit(1) resume_command = "gcloud scheduler jobs resume ratingservice-recollect-job --project {pid}".format( pid=project_id) _, err_str = Recipe._run_command(resume_command) if "ERROR:" in str(err_str, "utf-8"): print(err_str) logging.error("Failed executing service restoring command:" + err_str) else: print("...done") logging.info("Scheduled job of the rating service resumed")
def _deploy_state(state): """ Sets an environment variable CONVERT_CURRENCIES to given state and updates the state accordingly """ state_str = str(state).lower() set_env_command = f"kubectl set env deployment/frontend CONVERT_CURRENCIES={state_str}" get_pod_command = """kubectl get pod -l app=frontend -o \ jsonpath=\"{.items[0].metadata.name}\"""" logging.info('Setting env variable: %s', set_env_command) logging.info('Getting pod: %s', get_pod_command) Recipe._run_command(set_env_command) service, error = Recipe._run_command(get_pod_command) service = service.decode("utf-8").replace('"', '') if not service: print('No service found. Could not deploy state.') logging.error('No service found. Could not deploy state.') delete_pod_command = f"kubectl delete pod {service}" logging.info('Deleting pod: %s', delete_pod_command) Recipe._run_command(delete_pod_command) availability_command = "kubectl wait --for=condition=available --timeout=600s deployment/frontend" Recipe._run_command(availability_command)
def _deploy_state(state): """ Sets an environment variable CONVERT_CURRENCIES to given state and updates the state accordingly """ # Try getting Load Generator IP Recipe._auth_cluster(cluster="LOADGEN") get_loadgen_ip_command = """kubectl get service loadgenerator -o \ jsonpath=\"{.status.loadBalancer.ingress[0].ip}\"""" logging.info('Getting loadgen IP: %s', get_loadgen_ip_command) loadgen_ip, error = Recipe._run_command(get_loadgen_ip_command) if loadgen_ip is not None: loadgen_ip = loadgen_ip.decode("utf-8").replace('"', '').strip() if not loadgen_ip: operation = "start" if state else "stop" print( f"No load generator IP found. Will not {operation} synthetic load" ) logging.error( f"No load generator IP found. Will not {operation} synthetic load" ) elif not state: logging.info(f"Trying to stop any load generation.") resp = requests.post(f"http://{loadgen_ip}:81/api/stop") logging.info(f"Load Generator API Response: {resp.text}") Recipe._auth_cluster(cluster="APP") state_str = str(state).lower() set_env_command = f"kubectl set env deployment/frontend CONVERT_CURRENCIES={state_str}" get_pod_command = """kubectl get pod -l app=frontend -o \ jsonpath=\"{.items[0].metadata.name}\"""" logging.info('Setting env variable: %s', set_env_command) logging.info('Getting pod: %s', get_pod_command) Recipe._run_command(set_env_command) service, error = Recipe._run_command(get_pod_command) service = service.decode("utf-8").replace('"', '') if not service: print('No service found. Could not deploy state.') logging.error('No service found. Could not deploy state.') delete_pod_command = f"kubectl delete pod {service}" logging.info('Deleting pod: %s', delete_pod_command) Recipe._run_command(delete_pod_command) availability_command = "kubectl wait --for=condition=available --timeout=600s deployment/frontend" Recipe._run_command(availability_command) if loadgen_ip and state: # If we are breaking the service, start generating loads to the # frontend, after it is available, in order to expose the high # latency (caused by this recipe) in the frontend metrics. # The load will be generated by 20 users at spawn rate of 5 # users/seconds, and the load will auto-stop after 10 minutes. logging.info(f"Trying to start load generation") resp = requests.post( f"http://{loadgen_ip}:81/api/spawn/BasicHomePageViewingUser", { "user_count": 20, "spawn_rate": 5, "stop_after": 600 }) logging.info(f"Load Generator API Response: {resp.text}")