def incr_dns_count_metric(self, result):
"""
Helper method which increments the dns request count metric.
Args:
result: string of what happened - healthy, nxdomain etc
Returns: None, increments the statsd dns count metric
"""
result_data = {"result": result}
STATSD_CLIENT.incr(DNS_COUNT_METRIC_NAME % result_data)
def delete(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME %
self.action_data("delete")):
try:
LOGGER.info("Deleting Service %s from Namespace %s", self.name,
self.namespace)
self.api.delete_namespaced_service(
self.name,
self.namespace,
_request_timeout=e2e_globals.TEST_EVENT_TIMEOUTS)
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
self.incr_error_metric(error_code.name.lower())
LOGGER.error("Error deleting service %s from namespace %s",
self.name, self.namespace)
LOGGER.error("Error code: %s", error_code)
LOGGER.error("Error msg: %s", error_dict['message'])
LOGGER.debug("Error dict: %s", error_dict)
self.add_error(error_dict['message'])
except MaxRetryError:
LOGGER.error(
"Max retries exceeded when trying to delete service")
self.incr_error_metric("max_retries_exceeded")
self.add_error("max retries exceeded")
else:
self.wait_on_deleted()
super().delete(report)
def create(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME %
self.action_data("create")):
try:
returned_service = self.api.create_namespaced_service(
self.namespace, self.k8s_object)
LOGGER.debug(returned_service)
LOGGER.info("Service being created")
self.wait_on_event(self.api.list_namespaced_service,
e2e_globals.EventType.ADDED,
args=(self.namespace, ))
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
if error_code == HTTPStatus.UNPROCESSABLE_ENTITY:
LOGGER.error("Service object not valid - %s",
error_dict['message'])
if error_code == HTTPStatus.CONFLICT:
LOGGER.error("Service %s already exists in namespace %s",
self.name, self.namespace)
self.incr_error_metric(error_code.name.lower())
self.add_error(error_dict['message'])
LOGGER.debug(error_dict)
except MaxRetryError:
LOGGER.error(
"Max retries exceeded when trying to create service")
self.incr_error_metric("max_retries_exceeded")
self.add_error("Max retries exceeded")
super().create(report)
def create(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("create")):
try:
self.api.create_namespaced_config_map(self.namespace, self.k8s_object)
self.on_api = True
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
self.incr_error_metric(error_code.name.lower())
self.add_error(error_dict['message'])
if error_code == HTTPStatus.CONFLICT:
LOGGER.warning("ConfigMap already created, continuing")
elif error_code == HTTPStatus.NOT_FOUND:
not_found_msg = "namespaces \"{}\" not found".format(self.namespace)
if not_found_msg in error_dict['message']:
LOGGER.error("Namespace %s has disappeared, quitting", self.namespace)
sys.exit(error_dict['message'])
else:
LOGGER.error("Error making configmap %s: %s", error_code, error_dict['message'])
except MaxRetryError:
msg = "Error creating config map %s, max retries exceeded"
LOGGER.error(msg, self.name)
self.incr_error_metric("max_retries_exceeded")
self.add_error(msg % self.name)
else:
self.wait_on_event(self.api.list_namespaced_config_map, e2e_globals.EventType.ADDED,
args=(self.namespace,))
super().create(report)
def create(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("create")):
try:
self.extensions_api.create_namespaced_deployment(self.namespace, self.k8s_object)
self.on_api = True
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
self.incr_error_metric(error_code.name.lower())
self.add_error(error_dict['message'])
if error_code == HTTPStatus.CONFLICT:
LOGGER.warning("Deployment already exists, continuing")
LOGGER.debug("Error msg: %s", error_dict['message'])
elif error_code == HTTPStatus.FORBIDDEN:
LOGGER.error("Deployment creation forbidden - %s", error_dict['message'])
except MaxRetryError:
msg = "Error creating deployment %s, max retries exceeded"
self.add_error("max retries exceeded")
LOGGER.error(msg, self.name)
self.incr_error_metric("max_retries_exceeded")
else:
self.wait_on_event(self.extensions_api.list_namespaced_deployment, e2e_globals.EventType.ADDED,
args=(self.namespace,))
super().create(report)
def delete(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("delete")):
body = client.V1DeleteOptions(propagation_policy="Background")
try:
self.extensions_api.delete_namespaced_deployment(self.name, self.namespace, body)
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
msg = "Delete deployment %s from namespace %s failed, code: %s, msg: %s"
parameters = self.name, self.namespace, error_code.name.lower(), error_dict['message']
self.add_error(msg % parameters)
LOGGER.error(msg, *parameters)
self.incr_error_metric(error_code.name.lower())
except MaxRetryError:
msg = "Error deleting deployment %s from namespace %s, max retries exceeded"
parameters = self.name, self.namespace
self.add_error(msg % parameters)
LOGGER.error(msg, *parameters)
self.incr_error_metric("max_retries_exceeded")
else:
self.wait_on_deleted()
super().delete(report)
def make_http_request(self):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME %
self.action_data("http_get")):
response = None
if self._k8s_object is None:
self._read_from_k8s()
else:
url = "http://{}:{}".format(
self._k8s_object.status.pod_ip, self._k8s_object.spec.
containers[0].ports[0].container_port)
try:
response = requests.get(
url, headers=e2e_globals.TEST_REQUEST_HEADERS)
self.incr_http_count_metric(str(response.status_code))
except requests.HTTPError as e:
LOGGER.error(
"Pod %s at address: %s GET request failed: %s",
self.name, url, e)
self.incr_http_count_metric(str(response.status_code))
self.add_error("HTTP error code %i" % response.status_code)
except requests.ConnectionError as e:
LOGGER.error(
"Pod %s at address: %s GET request failed: %s",
self.name, url, e)
self.incr_http_count_metric("connection_error")
self.add_error(
"Pod %s at address: %s gave connection error" %
(self.name, url))
return response
def incr_http_count_metric(self, result, resource=None):
"""
Helper method which increments the http request count metric.
Args:
result: string of what happened - generally the HTTP status code
resource: resource name, defaults to the class name
Returns: None, increments the statsd http count metric
"""
result_data = {"result": result}
result_data.update(self.metric_data)
if resource is not None:
result_data["resource"] = resource
STATSD_CLIENT.incr(HTTP_COUNT_METRIC_NAME % result_data)
def make_http_request(self, hostname=False):
response = None
with STATSD_CLIENT.timer(ACTION_METRIC_NAME %
self.action_data("http_get")):
try:
svc = self.api.read_namespaced_service(
self.name,
self.namespace,
_request_timeout=e2e_globals.TEST_EVENT_TIMEOUTS)
if hostname:
ip = "{}.{}.svc.cluster.local".format(
self.name, self.namespace)
else:
ip = svc.spec.cluster_ip
port = svc.spec.ports[0].port
address = "http://{}:{}".format(ip, port)
try:
response = requests.get(
address, headers=e2e_globals.TEST_REQUEST_HEADERS)
LOGGER.info(
"Service %s at address: %s GET request response code: %s",
self.name, address, response.status_code)
LOGGER.debug(response)
self.incr_http_count_metric(str(response.status_code))
except requests.HTTPError as e:
LOGGER.error(
"Service %s at address: %s GET request failed: %s",
self.name, address, e)
self.incr_http_count_metric(str(response.status_code))
self.add_error("HTTP error code %i" % response.status_code)
except requests.ConnectionError as e:
LOGGER.error(
"Service %s at address: %s GET request failed: %s",
self.name, address, e)
self.incr_http_count_metric("connection_error")
self.add_error(
"Service %s at address: %s gave connection error" %
(self.name, address))
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
LOGGER.error(
"Error reading namespaced service in http service request test. Code: %s msg: %s",
error_code.name.lower(), error_dict['message'])
self.add_error(error_dict['message'])
self.incr_error_metric(error_code.name.lower())
return response
def incr_error_metric(self, error, area="api", resource=None):
"""
Helper method which takes in specific info about the data and
combines it with data about the class, then increases the statsd
metric
Args:
error: string representing what happened
area: string representing which part of the test the error occurred - should be one of:
- api (default)
- k8s
- timeout
see troubleshooting.md for more explanation of which one you might need to use.
resource: name of the resource being tested or being affected by the error. Default None
uses the classname
Returns: None, increments the statsd error metric
"""
error_data = {"area": area, "error": error}
error_data.update(self.metric_data)
if resource is not None:
error_data["resource"] = resource
STATSD_CLIENT.incr(ERROR_METRIC_NAME % error_data)
def delete(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("delete")):
try:
self.api.delete_namespaced_config_map(self.name, self.namespace, client.V1DeleteOptions())
self.on_api = False
except ApiException as e:
msg = "Error deleting config map %s - %s:%s"
error_code, error_dict = self.parse_error(e.body)
parameters = self.name, error_code, error_dict['message']
LOGGER.error(msg, *parameters)
self.add_error(msg % parameters)
self.incr_error_metric(error_code.name.lower())
else:
self.wait_on_deleted(report=False)
super().delete(report)
def delete(self, report=True):
with STATSD_CLIENT.timer(e2e_globals.ACTION_METRIC_NAME %
self.action_data("delete")):
try:
self.api.delete_namespaced_persistent_volume_claim(
self.name, self.namespace, client.V1DeleteOptions())
except ApiException as e:
error_code, error_dict = self.parse_error(e.body)
LOGGER.error(
"Error deleting volume claim %s, API exception: %s msg: %s",
self.name, error_code.name.lower(), error_dict['message'])
self.add_error(error_dict['message'])
self.incr_error_metric(error_code.name.lower())
else:
self.wait_on_deleted(report)
super().delete(report)
def create(self, report=True):
with STATSD_CLIENT.timer(e2e_globals.ACTION_METRIC_NAME %
self.action_data("create")):
try:
self.api.create_namespaced_persistent_volume_claim(
self.namespace, self.k8s_object)
except ApiException as e:
msg = "Error creating volume claim %s, API exception: %s msg: %s"
error_code, error_dict = self.parse_error(e.body)
LOGGER.error(msg, self.name, error_code.name.lower(),
error_dict['message'])
self.incr_error_metric(error_code.name.lower())
self.add_error(error_dict['message'])
else:
self.wait_on_event(
self.api.list_namespaced_persistent_volume_claim,
e2e_globals.EventType.ADDED,
args=(self.namespace, ))
self.on_api = True
super().create(report)
def watch_pod_scaling(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("scale", resource="Pod")):
self._wait_on_pods(event_type_enum=e2e_globals.EventType.DELETED)
if report:
self.send_update("Watch pod deletion")
def wait_on_pods_scheduled(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("schedule", resource="Pod")):
self._wait_on_pods(phase="Pending")
if report:
self.send_update("Wait on pod scheduling")
def wait_on_pods_ready(self, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("run", resource="Pod")):
self._wait_on_pods(phase="Running")
if report:
self.send_update("Wait on pods ready")
def scale(self, replicas, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("scale")):
self.replicas = replicas
self._update_k8s()
if report:
self.send_update("Scale deployment to %i replicas" % replicas)
def change_cfg_map(self, new_cfgmap_name, report=True):
with STATSD_CLIENT.timer(ACTION_METRIC_NAME % self.action_data("update")):
self.cfgmap_name = new_cfgmap_name
self._update_k8s()
if report:
self.send_update("Update deployment")