def hint(self): """ Provides a hint about finding the root cause of this recipe """ print('Giving hint for recipe') Recipe._auth_cluster() external_ip_command = "kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'" ip, error = Recipe._run_command(external_ip_command) ip = ip.decode("utf-8").replace("'", '') if not ip: print('No external IP found.') logging.error('No external IP found.') exit(1) print( 'Visit the external IP of the demo application to see if there are any visible changes: http://{}' .format(ip)) get_project_command = "gcloud config list --format value(core.project)" project_id, error = Recipe._run_command(get_project_command) project_id = project_id.decode("utf-8").replace('"', '') if not project_id: print('No project ID found.') logging.error('No project ID found.') exit(1) print( 'Use Monitoring Dashboards to see metrics associated with each service: https://console.cloud.google.com/monitoring/dashboards?project={}' .format(project_id)) print( 'Note: It may take up to 5 minutes for monitoring metrics to be updated' )
def restore_service(self): """ Rolls back the broken version of the given service and deploys the working version of the given service """ print('Deploying working service...') Recipe._auth_cluster() self._deploy_state(False) print('Done') logging.info('Deployed working service')
def break_service(self): """ Rolls back the working version of the given service and deploys the broken version of the given service """ print("Deploying broken service...") Recipe._auth_cluster() self.deploy_state(True) print("Done") logging.info('Deployed broken service')
def _deploy_state(state): """ Sets an environment variable CONVERT_CURRENCIES to given state and updates the state accordingly """ # Try getting Load Generator IP Recipe._auth_cluster(cluster="LOADGEN") get_loadgen_ip_command = """kubectl get service loadgenerator -o \ jsonpath=\"{.status.loadBalancer.ingress[0].ip}\"""" logging.info('Getting loadgen IP: %s', get_loadgen_ip_command) loadgen_ip, error = Recipe._run_command(get_loadgen_ip_command) if loadgen_ip is not None: loadgen_ip = loadgen_ip.decode("utf-8").replace('"', '').strip() if not loadgen_ip: operation = "start" if state else "stop" print( f"No load generator IP found. Will not {operation} synthetic load" ) logging.error( f"No load generator IP found. Will not {operation} synthetic load" ) elif not state: logging.info(f"Trying to stop any load generation.") resp = requests.post(f"http://{loadgen_ip}:81/api/stop") logging.info(f"Load Generator API Response: {resp.text}") Recipe._auth_cluster(cluster="APP") state_str = str(state).lower() set_env_command = f"kubectl set env deployment/frontend CONVERT_CURRENCIES={state_str}" get_pod_command = """kubectl get pod -l app=frontend -o \ jsonpath=\"{.items[0].metadata.name}\"""" logging.info('Setting env variable: %s', set_env_command) logging.info('Getting pod: %s', get_pod_command) Recipe._run_command(set_env_command) service, error = Recipe._run_command(get_pod_command) service = service.decode("utf-8").replace('"', '') if not service: print('No service found. Could not deploy state.') logging.error('No service found. Could not deploy state.') delete_pod_command = f"kubectl delete pod {service}" logging.info('Deleting pod: %s', delete_pod_command) Recipe._run_command(delete_pod_command) availability_command = "kubectl wait --for=condition=available --timeout=600s deployment/frontend" Recipe._run_command(availability_command) if loadgen_ip and state: # If we are breaking the service, start generating loads to the # frontend, after it is available, in order to expose the high # latency (caused by this recipe) in the frontend metrics. # The load will be generated by 20 users at spawn rate of 5 # users/seconds, and the load will auto-stop after 10 minutes. logging.info(f"Trying to start load generation") resp = requests.post( f"http://{loadgen_ip}:81/api/spawn/BasicHomePageViewingUser", { "user_count": 20, "spawn_rate": 5, "stop_after": 600 }) logging.info(f"Load Generator API Response: {resp.text}")