def delete_document(
name: str,
version_name: str = None,
force: bool = True,
configuration: Configuration = None,
secrets: Secrets = None,
) -> AWSResponse:
"""
creates a Systems Manager (SSM) document.
An SSM document defines the actions that SSM performs on your managed.
For more information about SSM documents:
https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-ssm-docs.html
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.create_document
"""
if not any(name):
raise ActivityFailed("To create a document," "you must specify the name")
try:
client = aws_client("ssm", configuration, secrets)
kwargs = {"Name": name, "Force": force}
if version_name:
kwargs["VersionName"] = version_name
return client.delete_document(**kwargs)
except ClientError as e:
raise ActivityFailed(
"Failed to delete document '{}': '{}'".format(
name, str(e.response["Error"]["Message"])
)
)
def microservice_available_and_healthy(
name: str,
ns: str = "default",
label_selector: str = "name in ({name})",
secrets: Secrets = None) -> Union[bool, None]:
"""
Lookup a deployment by `name` in the namespace `ns`.
The selected resources are matched by the given `label_selector`.
Raises :exc:`chaoslib.exceptions.ActivityFailed` when the state is not
as expected.
"""
label_selector = label_selector.format(name=name)
api = create_k8s_api_client(secrets)
v1 = client.AppsV1beta1Api(api)
ret = v1.list_namespaced_deployment(ns, label_selector=label_selector)
logger.debug("Found {d} deployments named '{n}'".format(d=len(ret.items),
n=name))
if not ret.items:
raise ActivityFailed(
"microservice '{name}' was not found".format(name=name))
for d in ret.items:
logger.debug("Deployment has '{s}' available replicas".format(
s=d.status.available_replicas))
if d.status.available_replicas != d.spec.replicas:
raise ActivityFailed(
"microservice '{name}' is not healthy".format(name=name))
return True
def delete_document(name: str,
version_name: str = None,
force: bool = True,
configuration: Configuration = None,
secrets: Secrets = None) -> AWSResponse:
"""
creates a Systems Manager (SSM) document.
An SSM document defines the actions that SSM performs on your managed.
For more information about SSM documents:
https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-ssm-docs.html
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.create_document
"""
if not any(name):
raise ActivityFailed('To create a document,'
'you must specify the name')
try:
client = aws_client('ssm', configuration, secrets)
return client.delete_document(Name=name,
VersionName=version_name,
Force=force)
except ClientError as e:
raise ActivityFailed("Failed to delete document '{}': '{}'".format(
name, str(e.response['Error']['Message'])))
def pods_in_phase(label_selector: str,
phase: str = "Running",
ns: str = "default",
secrets: Secrets = None) -> bool:
"""
Lookup a pod by `label_selector` in the namespace `ns`.
Raises :exc:`chaoslib.exceptions.ActivityFailed` when the state is not
as expected.
"""
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
ret = v1.list_namespaced_pod(ns, label_selector=label_selector)
logger.debug("Found {d} pods matching label '{n}'".format(
d=len(ret.items), n=label_selector))
if not ret.items:
raise ActivityFailed(
"no pods '{name}' were found".format(name=label_selector))
for d in ret.items:
if d.status.phase != phase:
raise ActivityFailed(
"pod '{name}' is in phase '{s}' but should be '{p}'".format(
name=label_selector, s=d.status.phase, p=phase))
return True
def pods_not_in_phase(
label_selector: str,
phase: str = "Running",
ns: str = "default",
secrets: Secrets = None,
) -> bool:
"""
Lookup a pod by `label_selector` in the namespace `ns`.
Raises :exc:`chaoslib.exceptions.ActivityFailed` when the pod is in the
given phase and should not have.
"""
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
if label_selector:
ret = v1.list_namespaced_pod(ns, label_selector=label_selector)
logger.debug(
f"Found {len(ret.items)} pods matching label '{label_selector}'"
f" in ns '{ns}'"
)
else:
ret = v1.list_namespaced_pod(ns)
logger.debug(f"Found {len(ret.items)} pods in ns '{ns}'")
if not ret.items:
raise ActivityFailed(f"no pods '{label_selector}' were found")
for d in ret.items:
if d.status.phase == phase:
raise ActivityFailed(
f"pod '{label_selector}' should not be in phase '{d.status.phase}'"
)
return True
def filter_instances(instances: List[OCIInstance] = None,
filters: Dict[str, Any] = None) -> List[OCIInstance]:
"""Return only those instances that match the filters provided."""
instances = instances or None
if instances is None:
raise ActivityFailed('No instances were found.')
filters_set = {x for x in filters}
available_filters_set = {x for x in instances[0].attribute_map}
# Partial filtering may return instances we do not want. We avoid it.
if not filters_set.issubset(available_filters_set):
raise ActivityFailed('Some of the chosen filters were not found,'
' we cannot continue.')
# Walk the instances and find those that match the given filters.
filtered = []
for instance in instances:
sentinel = True
for attr, val in filters.items():
if val != getattr(instance, attr, None):
sentinel = False
break
if sentinel:
filtered.append(instance)
return filtered
def create_document(path_content: str,
name: str,
version_name: str = None,
document_type: str = None,
document_format: str = None,
configuration: Configuration = None,
secrets: Secrets = None) -> AWSResponse:
"""
creates a Systems Manager (SSM) document.
An SSM document defines the actions that SSM performs on your managed.
For more information about SSM documents:
https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-ssm-docs.html
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.create_document
"""
if not any([path_content, name]):
raise ActivityFailed('To create a document,'
'you must specify the content and name')
try:
with open(path_content) as open_file:
document_content = open_file.read()
client = aws_client('ssm', configuration, secrets)
return client.create_document(Content=document_content,
Name=name,
VersionName=version_name,
DocumentType=document_type,
DocumentFormat=document_format)
except ClientError as e:
raise ActivityFailed("Failed to create document '{}': '{}'".format(
name, str(e.response['Error']['Message'])))
def validate_bucket_object_args(bucket_name, object_name):
"""
Ensure both arguments are valid ie not empty or raises error
:param bucket_name: Name of the bucket
:param object_name: Name of the object (as path format)
"""
if not bucket_name:
raise ActivityFailed("Cannot get object. " "Bucket name is mandatory.")
if not object_name:
raise ActivityFailed("Cannot get object. " "Object name is mandatory.")
def load_body(body_as_object: Dict[str, Any] = None,
body_as_yaml_file: str = None) -> Dict[str, Any]:
if (body_as_object is None) and (not body_as_yaml_file):
raise ActivityFailed(
"Either `body_as_object` or `body_as_yaml_file` must be set")
if body_as_object is not None:
return body_as_object
if not os.path.isfile(body_as_yaml_file):
raise ActivityFailed(
"Path '{}' is not a valid resource file".format(body_as_yaml_file))
else:
with open(body_as_yaml_file) as f:
return yaml.safe_load(f.read())
def uncordon_node(name: str = None,
label_selector: str = None,
secrets: Secrets = None):
"""
Uncordon nodes matching the given label name, so that pods can be
scheduled on them again.
"""
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
nodes = _select_nodes(name=name,
label_selector=label_selector,
secrets=secrets)
body = {"spec": {"unschedulable": False}}
for n in nodes:
try:
v1.patch_node(n.metadata.name, body)
except ApiException as x:
logger.debug("Scheduling node '{}' failed: {}".format(
n.metadata.name, x.body))
raise ActivityFailed("Failed to schedule node '{}': {}".format(
n.metadata.name, x.body))
def patch_cluster_custom_object(group: str, version: str, plural: str,
name: str, force: bool = False,
resource: Dict[str, Any] = None,
resource_as_yaml_file: str = None,
secrets: Secrets = None) -> Dict[str, Any]:
"""
Patch a custom object cluster-wide. The resource must be the
updated version to apply. Force will re-acquire conflicting fields
owned by others.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
body = load_body(resource, resource_as_yaml_file)
try:
r = api.patch_cluster_custom_object(
group, version, plural, name, body, force=force,
_preload_content=False
)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
"Failed to patch custom resource object: '{}' {}".format(
x.reason, x.body))
def list_domains(fqdn_filter: str = None,
tld_filter: str = None,
configuration: Configuration = None,
secrets: Secrets = None) -> List[Dict[str, Any]]:
"""
List all domains or those matching the given TLD or FQDN filters and
return the list as-is.
See https://api.gandi.net/docs/domains/#v5-domain-domains
"""
with gandi_client(configuration, secrets) as client:
url = gandi_url("/v5/domain/domains")
qs = {}
if fqdn_filter:
qs["fqdn"] = fqdn_filter
if tld_filter:
qs["tld"] = tld_filter
if qs:
url = "{}?{}".format(url, urlencode(qs))
r = client.get(url)
if r.status_code > 399:
raise ActivityFailed(
"Failed to list domains from Gandi: {}".format(r.text))
domains = r.json()
logger.debug("Gandi domains: {}".format(domains))
return domains
def deployment_not_fully_available(
name: str,
ns: str = "default",
label_selector: str = None,
timeout: int = 30,
secrets: Secrets = None) -> Union[bool, None]:
"""
Wait until the deployment gets into an intermediate state where not all
expected replicas are available. Once this state is reached, return `True`.
If the state is not reached after `timeout` seconds, a
:exc:`chaoslib.exceptions.ActivityFailed` exception is raised.
"""
if _deployment_readiness_has_state(
name,
False,
ns,
label_selector,
timeout,
secrets,
):
return True
else:
raise ActivityFailed(
"deployment '{name}' failed to stop running within {t}s".format(
name=name, t=timeout))
def statefulset_not_fully_available(
name: str,
ns: str = "default",
label_selector: str = None,
timeout: int = 30,
secrets: Secrets = None,
):
"""
Wait until the statefulSet gets into an intermediate state where not all
expected replicas are available. Once this state is reached, return `True`.
If the state is not reached after `timeout` seconds, a
:exc:`chaoslib.exceptions.ActivityFailed` exception is raised.
"""
if _statefulset_readiness_has_state(
name,
False,
ns,
label_selector,
timeout,
secrets,
):
return True
else:
raise ActivityFailed(
f"microservice '{name}' failed to stop running within {timeout}s")
def domains_should_not_expire_in(value: List[Dict[str, Any]] = None,
when: str = '1 month') -> bool:
"""
Go through the list of Gandi domains and fails if any expires before
the given date threshold as a relative time to now.
"""
no_sooner_than_date = parse('in {}'.format(when), settings=TZ_SETTINGS)
if not no_sooner_than_date:
raise ActivityFailed(
"Failed to parse `when` date '{}' for Gandi domain expire "
"tolerance".format(when))
logger.debug(
"Looking through domains to see if any expires before '{}'".format(
no_sooner_than_date))
all_valid = True
for domain in value:
expire_date = parse(domain["dates"]["registry_ends_at"],
settings=TZ_SETTINGS)
if expire_date < no_sooner_than_date:
logger.warning("Domain '{}' expires in less than '{}': {}".format(
domain["fqdn_unicode"], when,
domain["dates"]["registry_ends_at"]))
all_valid = False
return all_valid
def get_custom_object(
group: str,
version: str,
plural: str,
name: str,
ns: str = "default",
secrets: Secrets = None,
) -> Dict[str, Any]:
"""
Get a custom object in the given namespace.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
try:
r = api.get_namespaced_custom_object(group,
version,
ns,
plural,
name,
_preload_content=False)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
f"Failed to create custom resource object: '{x.reason}' {x.body}")
def create_cluster_custom_object(group: str, version: str, plural: str,
resource: Dict[str, Any] = None,
resource_as_yaml_file: str = None,
secrets: Secrets = None) -> Dict[str, Any]:
"""
Delete a custom object in the given namespace.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
body = load_body(resource, resource_as_yaml_file)
try:
r = api.create_cluster_custom_object(
group, version, plural, body, _preload_content=False
)
return json.loads(r.data)
except ApiException as x:
if x.status == 409:
logger.debug(
"Custom resource object {}/{} already exists".format(
group, version))
return json.loads(x.body)
else:
raise ActivityFailed(
"Failed to create custom resource object: '{}' {}".format(
x.reason, x.body))
def replace_cluster_custom_object(group: str, version: str, plural: str,
name: str, force: bool = False,
resource: Dict[str, Any] = None,
resource_as_yaml_file: str = None,
secrets: Secrets = None) -> Dict[str, Any]:
"""
Replace a custom object in the given namespace. The resource must be the
new version to apply.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
body = load_body(resource, resource_as_yaml_file)
try:
r = api.replace_cluster_custom_object(
group, version, plural, name, body, force=force,
_preload_content=False
)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
"Failed to replace custom resource object: '{}' {}".format(
x.reason, x.body))
def get_all_events(from_time: str, to_time: str, configuration: Configuration,
secrets: Secrets) -> InstanaResponse:
"""
Get all events from instana within a time window, given by the from_time
and the to_time for details of the api see
https://instana.github.io/openapi/#tag/Events
"""
logger.debug("get_all_events")
instana_host = configuration.get("instana_host")
instana_api_token = secrets.get("instana_api_token")
if not (instana_host and instana_api_token):
raise ActivityFailed(
"No Instana Host or API Token Secrete were found.")
url = "{}/api/events".format(configuration.get("instana_host"))
params = {}
if from_time:
params["from"] = from_time
if to_time:
params["to"] = to_time
result = execute_instana_get_request(url, params, secrets)
return result
def count_instances(filters: List[Dict[str, Any]],
compartment_id: str = None,
configuration: Configuration = None,
secrets: Secrets = None) -> int:
"""
Return the number of instances in accordance with the given filters.
Please refer to: https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/core/models/oci.core.models.Instance.html#oci.core.models.Instance
for details on the available filters under the 'parameters' section.
""" # noqa: E501
compartment_id = compartment_id or from_file().get('compartment')
if compartment_id is None:
raise ActivityFailed('We have not been able to find a compartment,'
' without one, we cannot continue.')
client = oci_client(ComputeClient,
configuration,
secrets,
skip_deserialization=False)
filters = filters or None
instances = get_instances(client, compartment_id)
if filters is not None:
return len(filter_instances(instances, filters=filters))
return len(instances)
def delete_custom_object(
group: str,
version: str,
plural: str,
name: str,
ns: str = "default",
secrets: Secrets = None,
) -> Dict[str, Any]:
"""
Create a custom object cluster wide. Its custom resource
definition must already exists or this will fail with a 404.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
try:
r = api.delete_namespaced_custom_object(group,
version,
ns,
plural,
name,
_preload_content=False)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
f"Failed to delete custom resource object: '{x.reason}' {x.body}")
def create_node(meta: Dict[str, Any] = None,
spec: Dict[str, Any] = None,
secrets: Secrets = None) -> client.V1Node:
"""
Create one new node in the cluster.
Due to the way things work on certain cloud providers, you won't be able
to use this meaningfully on them. For instance on GCE, this will likely
fail.
See also: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#idempotency
""" # noqa: E501
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
body = client.V1Node()
body.metadata = client.V1ObjectMeta(**meta) if meta else None
body.spec = client.V1NodeSpec(**spec) if spec else None
try:
res = v1.create_node(body)
except ApiException as x:
raise ActivityFailed("Creating new node failed: {}".format(x.body))
logger.debug("Node '{}' created".format(res.metadata.name))
return res
def all_microservices_healthy(ns: str = "default",
secrets: Secrets = None) -> MicroservicesStatus:
"""
Check all microservices in the system are running and available.
Raises :exc:`chaoslib.exceptions.ActivityFailed` when the state is not
as expected.
"""
api = create_k8s_api_client(secrets)
not_ready = []
failed = []
v1 = client.CoreV1Api(api)
ret = v1.list_namespaced_pod(namespace=ns)
for p in ret.items:
phase = p.status.phase
if phase == "Failed":
failed.append(p)
elif phase not in ("Running", "Succeeded"):
not_ready.append(p)
logger.debug("Found {d} failed and {n} not ready pods".format(
d=len(failed), n=len(not_ready)))
# we probably should list them in the message
if failed or not_ready:
raise ActivityFailed("the system is unhealthy")
return True
def microservice_is_not_available(name: str,
ns: str = "default",
label_selector: str = "name in ({name})",
secrets: Secrets = None) -> bool:
"""
Lookup pods with a `name` label set to the given `name` in the specified
`ns`.
Raises :exc:`chaoslib.exceptions.ActivityFailed` when one of the pods
with the specified `name` is in the `"Running"` phase.
"""
label_selector = label_selector.format(name=name)
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
if label_selector:
ret = v1.list_namespaced_pod(ns, label_selector=label_selector)
else:
ret = v1.list_namespaced_pod(ns)
logger.debug("Found {d} pod(s) named '{n}' in ns '{s}".format(d=len(
ret.items),
n=name,
s=ns))
for p in ret.items:
phase = p.status.phase
logger.debug("Pod '{p}' has status '{s}'".format(p=p.metadata.name,
s=phase))
if phase == "Running":
raise ActivityFailed(
"microservice '{name}' is actually running".format(name=name))
return True
def service_endpoint_is_initialized(name: str,
ns: str = "default",
label_selector: str = "name in ({name})",
secrets: Secrets = None):
"""
Lookup a service endpoint by its name and raises :exc:`FailedProbe` when
the service was not found or not initialized.
"""
label_selector = label_selector.format(name=name)
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
if label_selector:
ret = v1.list_namespaced_service(ns, label_selector=label_selector)
else:
ret = v1.list_namespaced_service(ns)
logger.debug("Found {d} service(s) named '{n}' ins ns '{s}'".format(d=len(
ret.items),
n=name,
s=ns))
if not ret.items:
raise ActivityFailed(
"service '{name}' is not initialized".format(name=name))
return True
def deployment_fully_available(
name: str,
ns: str = "default",
label_selector: str = None,
timeout: int = 30,
secrets: Secrets = None,
) -> Union[bool, None]:
"""
Wait until all the deployment expected replicas are available.
Once this state is reached, return `True`.
If the state is not reached after `timeout` seconds, a
:exc:`chaoslib.exceptions.ActivityFailed` exception is raised.
"""
if _deployment_readiness_has_state(
name,
True,
ns,
label_selector,
timeout,
secrets,
):
return True
else:
raise ActivityFailed(
f"deployment '{name}' failed to recover within {timeout}s")
def revoke_security_group_ingress(requested_security_group_id: str,
ip_protocol: str,
from_port: int,
to_port: int,
ingress_security_group_id: str = None,
cidr_ip: str = None,
configuration: Configuration = None,
secrets: Secrets = None) -> AWSResponse:
"""
Remove one ingress rule from a security group
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.revoke_security_group_ingress
- requested_security_group_id: the id for the security group to update
- ip_protocol: ip protocol name (tcp, udp, icmp, icmpv6) or -1 to specify all
- from_port: start of port range
- to_port: end of port range
- ingress_security_group_id: id of the securiy group to allow access to. You can either specify this or cidr_ip.
- cidr_ip: the IPv6 CIDR range. You can either specify this or ingress_security_group_id
""" # noqa: E501
client = aws_client('ec2', configuration, secrets)
request_kwargs = create_ingress_kwargs(
requested_security_group_id,
ip_protocol,
from_port,
to_port,
ingress_security_group_id,
cidr_ip
)
try:
response = client.revoke_security_group_ingress(**request_kwargs)
return response
except ClientError as e:
raise ActivityFailed(
'Failed to remove ingress rule: {}'.format(
e.response["Error"]["Message"]))
def run_process_activity(activity: Activity, configuration: Configuration,
secrets: Secrets) -> Any:
"""
Run the a process activity.
A process activity is an executable the current user is allowed to apply.
The raw result of that command is returned as bytes of this activity.
Raises :exc:`ActivityFailed` when a the process takes longer than the
timeout defined in the activity. There is no timeout by default so be
careful when you do not explicitly provide one.
This should be considered as a private function.
"""
provider = activity["provider"]
timeout = provider.get("timeout", None)
arguments = provider.get("arguments", [])
if arguments and (configuration or secrets):
arguments = substitute(arguments, configuration, secrets)
shell = False
path = shutil.which(os.path.expanduser(provider["path"]))
if isinstance(arguments, str):
shell = True
arguments = "{} {}".format(path, arguments)
else:
if isinstance(arguments, dict):
arguments = itertools.chain.from_iterable(arguments.items())
arguments = list([str(p) for p in arguments if p not in (None, "")])
arguments.insert(0, path)
try:
logger.debug("Running: {a}".format(a=str(arguments)))
proc = subprocess.run(arguments,
timeout=timeout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ,
shell=shell)
except subprocess.TimeoutExpired:
raise ActivityFailed("process activity took too long to complete")
# kind warning to the user that this process returned a non--zero
# exit code, as traditionally used to indicate a failure,
# but not during the hypothesis check because that could also be
# exactly what the user want. This warning is helpful during the
# method and rollbacks
if "tolerance" not in activity and proc.returncode > 0:
logger.warning(
"This process returned a non-zero exit code. "
"This may indicate some error and not what you expected. "
"Please have a look at the logs.")
stdout = decode_bytes(proc.stdout)
stderr = decode_bytes(proc.stderr)
return {"status": proc.returncode, "stdout": stdout, "stderr": stderr}
def delete_nodes(label_selector: str = None, all: bool = False,
rand: bool = False, count: int = None,
grace_period_seconds: int = None, secrets: Secrets = None):
"""
Delete nodes gracefully. Select the appropriate nodes by label.
Nodes are not drained beforehand so we can see how cluster behaves. Nodes
cannot be restarted, they are really deleted. Please be careful when using
this action.
On certain cloud providers, you also need to delete the underneath VM
instance as well afterwards. This is the case on GCE for instance.
If `all` is set to `True`, all nodes will be terminated.
If `rand` is set to `True`, one random node will be terminated.
If ̀`count` is set to a positive number, only a upto `count` nodes
(randomly picked) will be terminated. Otherwise, the first retrieved node
will be terminated.
"""
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
ret = v1.list_node(label_selector=label_selector)
logger.debug("Found {d} nodes labelled '{s}'".format(
d=len(ret.items), s=label_selector))
nodes = ret.items
if not nodes:
raise ActivityFailed(
"failed to find a node that matches selector {}".format(
label_selector))
if rand:
nodes = [random.choice(nodes)]
logger.debug("Picked node '{p}' to be terminated".format(
p=nodes[0].metadata.name))
elif count is not None:
nodes = random.choices(nodes, k=count)
logger.debug("Picked {c} nodes '{p}' to be terminated".format(
c=len(nodes), p=", ".join([n.metadata.name for n in nodes])))
elif not all:
nodes = [nodes[0]]
logger.debug("Picked node '{p}' to be terminated".format(
p=nodes[0].metadata.name))
else:
logger.debug("Picked all nodes '{p}' to be terminated".format(
p=", ".join([n.metadata.name for n in nodes])))
body = client.V1DeleteOptions()
for n in nodes:
res = v1.delete_node(
n.metadata.name, body, grace_period_seconds=grace_period_seconds)
if res.status != "Success":
logger.debug("Terminating nodes failed: {}".format(res.message))
def failure_rate(entity: str, relative_time: str,
failed_percentage: int, configuration: Configuration,
secrets: Secrets = None) -> bool:
"""
Validates the failure rate of a specific service.
Returns true if the failure rate is less than the expected failure rate
For more information check the api documentation.
https://www.dynatrace.com/support/help/dynatrace-api/environment-api/metric-v1/
"""
dynatrace = configuration.get("dynatrace")
if not any([dynatrace]):
raise ActivityFailed('To run commands,'
'you must specify the dynatrace api and token '
'in the configuration section')
base = dynatrace.get("dynatrace_base_url")
url = "{base}/api/v1/timeseries".format(base=base)
params = {"timeseriesId": "com.dynatrace.builtin:service.failurerate",
"relativeTime": relative_time,
"aggregationType": "AVG",
"entity": entity}
r = requests.get(url,
headers={"Content-Type": "application/json",
"Authorization": "Api-Token " +
dynatrace.get("dynatrace_token")},
params=params)
if r.status_code != 200:
raise ActivityFailed(
"Dynatrace query {q} failed: {m}".format(q=str(params), m=r.text))
acum = 0
count = 0
for x in r.json().get("result").get("dataPoints").get(entity):
if x[1] is not None:
acum = acum + x[1]
count = count+1
logger.debug("faile rate percentage '{}'".format((acum/count)))
if (acum/count) < failed_percentage:
return True
return False