def _update_and_schedule(self, deploy: kube.MANIFEST) -> List[Event]:
"""
Update (patch) the Deployment and schedule the next Event.
This assumes proper changes have been made to the Deployment manifest
and the next times for each lifecycle event have been set in the proper
annotation.
"""
anno = deploy["metadata"]["annotations"]
destroy_time = float(anno["ocs-monkey/osio-destroy-at"])
idle_time = float(anno["ocs-monkey/osio-idle-at"])
health_time = float(anno["ocs-monkey/osio-health-at"])
next_time = min(destroy_time, idle_time, health_time)
anno["ocs-monkey/osio-next-time"] = str(next_time)
if next_time == destroy_time:
anno["ocs-monkey/osio-next-action"] = "destroy"
elif next_time == idle_time:
anno["ocs-monkey/osio-next-action"] = "idle"
else:
anno["ocs-monkey/osio-next-action"] = "health"
apps_v1 = k8s.AppsV1Api()
kube.call(apps_v1.patch_namespaced_deployment,
namespace=self._namespace,
name=self._name,
body=deploy)
return [Lifecycle(next_time, self._namespace, self._name)]
def invoke(self) -> None:
core_v1 = k8s.CoreV1Api()
kube.call(core_v1.delete_namespaced_pod,
namespace=self._namespace,
name=self._name,
grace_period_seconds=0,
body=k8s.V1DeleteOptions())
def execute(self) -> 'List[Event]':
"""Create a new Deployment & schedule it's destruction."""
destroy_time = time.time() + random.expovariate(1 / self._lifetime)
manifests = _get_workload(self._namespace, self._storage_class,
self._access_mode)
pvc = manifests["pvc"]
deploy = manifests["deployment"]
# Set necessary accotations on the Deployment
anno = deploy["metadata"].setdefault("annotations", {})
anno["ocs-monkey/osio-active"] = str(self._active)
anno["ocs-monkey/osio-idle"] = str(self._idle)
anno["ocs-monkey/osio-destroy-at"] = str(destroy_time)
anno["ocs-monkey/osio-pvc"] = pvc["metadata"]["name"]
deploy["metadata"]["annotations"] = anno
LOGGER.info("Create: %s/%s, %s", deploy["metadata"]["namespace"],
deploy["metadata"]["name"], pvc["metadata"]["name"])
core_v1 = k8s.CoreV1Api()
kube.call(core_v1.create_namespaced_persistent_volume_claim,
namespace=pvc["metadata"]["namespace"],
body=pvc)
apps_v1 = k8s.AppsV1Api()
kube.call(apps_v1.create_namespaced_deployment,
namespace=deploy["metadata"]["namespace"],
body=deploy)
EXECUTOR.submit(_pod_start_watcher, deploy)
return [
Lifecycle(
when=0, # execute asap
namespace=deploy["metadata"]["namespace"],
name=deploy["metadata"]["name"],
),
Creator(self._namespace, self._storage_class, self._access_mode,
self._interarrival, self._lifetime, self._active,
self._idle)
]
def unique_namespace(request, load_kubeconfig):
"""
Create a namespace in which to run a test.
This will create a namespace with a random name and automatically delete it
at the end of the test.
Returns:
dict describing the namespace that has been created for this test.
"""
core_v1 = k8s.CoreV1Api()
ns_name = f"ns-{random.randrange(999999999)}"
namespace = kube.call(core_v1.create_namespace,
body={"metadata": {
"name": ns_name
}})
def teardown():
kube.call(core_v1.delete_namespace,
name=namespace["metadata"]["name"],
body=k8s.V1DeleteOptions())
request.addfinalizer(teardown)
return namespace
def resume(namespace: str) -> List[Event]:
"""
Re-adopt deployments that were previously created.
If the workload generator exits, Deployments that were created from the
previous instance will still be present in the cluster. If the workload
generator is subsequently restarted, this function can be used to locate and
"adopt" those alread-present deployments, resuming their scheduled lifecycle
events.
Some lifecycle events may have been missed while the generator was down, but
they will be processed immediately once the deployments are adopted.
Parameters:
namespace: The namespace containing the deployments
Returns:
a list of Events to be enqueued onto the Dispatcher
"""
events: List[Event] = []
apps_v1 = k8s.AppsV1Api()
deployments = kube.call(apps_v1.list_namespaced_deployment,
namespace=namespace,
label_selector='ocs-monkey/controller=osio')
for deployment in deployments["items"]:
LOGGER.info("Found: %s/%s", deployment["metadata"]["namespace"],
deployment["metadata"]["name"])
events.append(Lifecycle(when=0,
namespace=deployment["metadata"]["namespace"],
name=deployment["metadata"]["name"]))
return events
def _action_destroy(self, deploy: kube.MANIFEST) -> None:
anno = deploy["metadata"]["annotations"]
pvc_name = anno["ocs-monkey/osio-pvc"]
LOGGER.info("Destroy: %s/%s, %s", self._namespace, self._name,
pvc_name)
EXECUTOR.submit(_pod_stop_watcher, copy.deepcopy(deploy))
apps_v1 = k8s.AppsV1Api()
kube.call(apps_v1.delete_namespaced_deployment,
namespace=self._namespace,
name=self._name,
body=k8s.V1DeleteOptions())
core_v1 = k8s.CoreV1Api()
kube.call(core_v1.delete_namespaced_persistent_volume_claim,
namespace=self._namespace,
name=pvc_name,
body=k8s.V1DeleteOptions())
def _get_cephcluster(self) -> kube.MANIFEST:
crd = k8s.CustomObjectsApi()
return kube.call(crd.get_namespaced_custom_object,
group="ceph.rook.io",
version="v1",
plural="cephclusters",
namespace=self._ns,
name=self._name)
def get(self) -> Failure:
# This is overly restrictive. We should be looking at
# self._cluster.problems() and taking into account the type of failure.
if not self._cluster.is_healthy():
raise NoSafeFailures("ceph cluster is not healthy")
selector = ','.join([f'{key}={val}' for (key, val) in
self._labels.items()])
apps_v1 = k8s.AppsV1Api()
deployments = kube.call(apps_v1.list_namespaced_deployment,
namespace=self._namespace,
label_selector=selector)
if not deployments["items"]:
raise NoSafeFailures(f'No deployments matched selector: {selector}')
# If any of the selected Deployments are degraded, stop. This is because
# each component has separate Deployments per replica. E.g., MONs are 3
# separate deployments.
for deployment in deployments["items"]:
if deployment["spec"]["replicas"] != deployment["status"].get("ready_replicas"):
raise NoSafeFailures('No pods are safe to kill')
random.shuffle(deployments["items"])
deployment = deployments["items"][0]
pod_selector = ','.join([f'{key}={val}' for (key, val) in
deployment["spec"]["selector"]["match_labels"].items()])
core_v1 = k8s.CoreV1Api()
pods = kube.call(core_v1.list_namespaced_pod,
namespace=self._namespace,
label_selector=pod_selector)
if not pods["items"]:
raise NoSafeFailures(f'No pods maatched selector: {pod_selector}')
random.shuffle(pods["items"])
return DeletePod(deployment, pods["items"][0])
def teardown():
kube.call(core_v1.delete_namespace,
name=namespace["metadata"]["name"],
body=k8s.V1DeleteOptions())
def _delete_namespace(ns_name: str) -> None:
core_v1 = k8s.CoreV1Api()
kube.call(core_v1.delete_namespace,
name=ns_name,
body=k8s.V1DeleteOptions())
def _get_deployment(self) -> kube.MANIFEST:
apps_v1 = k8s.AppsV1Api()
v1dl = kube.call(apps_v1.list_namespaced_deployment,
namespace=self._namespace,
field_selector=f'metadata.name={self._name}')
return v1dl["items"][0] # type: ignore