def hint(self):
"""
Provides a hint about finding the root cause of this recipe
"""
print('Giving hint for recipe')
Recipe._auth_cluster()
external_ip_command = "kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'"
ip, error = Recipe._run_command(external_ip_command)
ip = ip.decode("utf-8").replace("'", '')
if not ip:
print('No external IP found.')
logging.error('No external IP found.')
exit(1)
print(
'Visit the external IP of the demo application to see if there are any visible changes: http://{}'
.format(ip))
get_project_command = "gcloud config list --format value(core.project)"
project_id, error = Recipe._run_command(get_project_command)
project_id = project_id.decode("utf-8").replace('"', '')
if not project_id:
print('No project ID found.')
logging.error('No project ID found.')
exit(1)
print(
'Use Monitoring Dashboards to see metrics associated with each service: https://console.cloud.google.com/monitoring/dashboards?project={}'
.format(project_id))
print(
'Note: It may take up to 5 minutes for monitoring metrics to be updated'
)
def hint(self):
"""
Provides a hint about the root cause of the issue
"""
get_project_command = "gcloud config list --format value(core.project)"
project_id, error = Recipe._run_command(get_project_command)
project_id = project_id.decode("utf-8").replace('"', '')
if not project_id:
print('No project ID found.')
logging.error('No project ID found.')
exit(1)
print(
'Use Cloud Logging to view logs exported by each service: https://console.cloud.google.com/logs?project={}'
.format(project_id))
def restore_service(self):
"""
Resume Cloud scheduler job to restore calls to recollect API of
the rating service
"""
logging.info("Resuming scheduled job of the rating service")
print("Restoring broken operations...")
project_id = Recipe._get_project_id()
if not project_id:
print("Failed: cannot find project id.")
logging.error("Failed pausing scheduled job: no project id.")
exit(1)
resume_command = "gcloud scheduler jobs resume ratingservice-recollect-job --project {pid}".format(
pid=project_id)
_, err_str = Recipe._run_command(resume_command)
if "ERROR:" in str(err_str, "utf-8"):
print(err_str)
logging.error("Failed executing service restoring command:" +
err_str)
else:
print("...done")
logging.info("Scheduled job of the rating service resumed")
def _deploy_state(state):
"""
Sets an environment variable CONVERT_CURRENCIES to given state
and updates the state accordingly
"""
state_str = str(state).lower()
set_env_command = f"kubectl set env deployment/frontend CONVERT_CURRENCIES={state_str}"
get_pod_command = """kubectl get pod -l app=frontend -o \
jsonpath=\"{.items[0].metadata.name}\""""
logging.info('Setting env variable: %s', set_env_command)
logging.info('Getting pod: %s', get_pod_command)
Recipe._run_command(set_env_command)
service, error = Recipe._run_command(get_pod_command)
service = service.decode("utf-8").replace('"', '')
if not service:
print('No service found. Could not deploy state.')
logging.error('No service found. Could not deploy state.')
delete_pod_command = f"kubectl delete pod {service}"
logging.info('Deleting pod: %s', delete_pod_command)
Recipe._run_command(delete_pod_command)
availability_command = "kubectl wait --for=condition=available --timeout=600s deployment/frontend"
Recipe._run_command(availability_command)
def _deploy_state(state):
"""
Sets an environment variable CONVERT_CURRENCIES to given state
and updates the state accordingly
"""
# Try getting Load Generator IP
Recipe._auth_cluster(cluster="LOADGEN")
get_loadgen_ip_command = """kubectl get service loadgenerator -o \
jsonpath=\"{.status.loadBalancer.ingress[0].ip}\""""
logging.info('Getting loadgen IP: %s', get_loadgen_ip_command)
loadgen_ip, error = Recipe._run_command(get_loadgen_ip_command)
if loadgen_ip is not None:
loadgen_ip = loadgen_ip.decode("utf-8").replace('"', '').strip()
if not loadgen_ip:
operation = "start" if state else "stop"
print(
f"No load generator IP found. Will not {operation} synthetic load"
)
logging.error(
f"No load generator IP found. Will not {operation} synthetic load"
)
elif not state:
logging.info(f"Trying to stop any load generation.")
resp = requests.post(f"http://{loadgen_ip}:81/api/stop")
logging.info(f"Load Generator API Response: {resp.text}")
Recipe._auth_cluster(cluster="APP")
state_str = str(state).lower()
set_env_command = f"kubectl set env deployment/frontend CONVERT_CURRENCIES={state_str}"
get_pod_command = """kubectl get pod -l app=frontend -o \
jsonpath=\"{.items[0].metadata.name}\""""
logging.info('Setting env variable: %s', set_env_command)
logging.info('Getting pod: %s', get_pod_command)
Recipe._run_command(set_env_command)
service, error = Recipe._run_command(get_pod_command)
service = service.decode("utf-8").replace('"', '')
if not service:
print('No service found. Could not deploy state.')
logging.error('No service found. Could not deploy state.')
delete_pod_command = f"kubectl delete pod {service}"
logging.info('Deleting pod: %s', delete_pod_command)
Recipe._run_command(delete_pod_command)
availability_command = "kubectl wait --for=condition=available --timeout=600s deployment/frontend"
Recipe._run_command(availability_command)
if loadgen_ip and state:
# If we are breaking the service, start generating loads to the
# frontend, after it is available, in order to expose the high
# latency (caused by this recipe) in the frontend metrics.
# The load will be generated by 20 users at spawn rate of 5
# users/seconds, and the load will auto-stop after 10 minutes.
logging.info(f"Trying to start load generation")
resp = requests.post(
f"http://{loadgen_ip}:81/api/spawn/BasicHomePageViewingUser", {
"user_count": 20,
"spawn_rate": 5,
"stop_after": 600
})
logging.info(f"Load Generator API Response: {resp.text}")
