def create(self):
# Check to see if this isn't meant to deploy first
if not self.__config["deploy"]:
return
evaluate_request(
self.__custom_objects_api.create_namespaced_custom_object(
namespace=self.__namespace,
body=self.redis_metrics,
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
),
allowed_statuses=[409],
)
evaluate_request(
self.__custom_objects_api.create_namespaced_custom_object(
namespace=self.__namespace,
body=self.bps_metrics,
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
),
allowed_statuses=[409],
)
def delete(self):
"""Removes the Logstash resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_config_map(
namespace=self.__namespace,
name=get_name(self.logstash_conf),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_config_map(
namespace=self.__namespace,
name=get_name(self.logstash_yml),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.logstash_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_stateful_set(
namespace=self.__namespace,
name=get_name(self.logstash),
async_req=True,
propagation_policy="Background",
))
def create(self):
"""Creates the Logstash resources in the cluster
"""
evaluate_request(
self.__v1_api.create_namespaced_config_map(
namespace=self.__namespace,
body=self.logstash_conf,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_api.create_namespaced_config_map(
namespace=self.__namespace,
body=self.logstash_yml,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_api.create_namespaced_service(namespace=self.__namespace,
body=self.logstash_service,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_stateful_set(
namespace=self.__namespace, body=self.logstash,
async_req=True),
allowed_statuses=[409],
)
def delete(self):
"""Removes the load generation resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.load_gen_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.load_gen_deployment),
async_req=True,
propagation_policy="Background",
))
def initialize_namespace(namespace,
store_count=250,
customers=5000,
simulation=2500):
"""Initialize namespace and config map for kapture
Sets up the namespace and config map for Kapture to use. If they already exist, this method
will quietly do nothing.
"""
api = client.CoreV1Api()
evaluate_request(
api.create_namespace(
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": namespace
}
},
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
api.create_namespaced_config_map(
namespace=namespace,
body={
"apiVersion": "v1",
"data": {
"STORE_COUNT": str(store_count),
"CUSTOMERS": str(customers),
"SIMULATION": str(simulation),
"BPS_TOPIC": "bps-data",
},
"kind": "ConfigMap",
"metadata": {
"name": "kapture-config",
"namespace": namespace
},
},
async_req=True,
),
allowed_statuses=[409],
)
def delete(self):
evaluate_request(
self.__custom_objects_api.delete_namespaced_custom_object(
namespace=self.__namespace,
body={},
name=get_name(self.redis_metrics),
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
))
evaluate_request(
self.__custom_objects_api.delete_namespaced_custom_object(
namespace=self.__namespace,
body={},
name=get_name(self.bps_metrics),
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
))
def create(self):
"""Creates the load generation resources in the cluster
"""
evaluate_request(
self.__v1_api.create_namespaced_service(namespace=self.__namespace,
body=self.load_gen_service,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace,
body=self.load_gen_deployment,
async_req=True,
),
allowed_statuses=[409],
)
# Make a second call to patch the configuration to make sure changes in replicas are applied
# TODO: Investigate making a utility method similar to `kubectl apply`
evaluate_request(
self.__v1_apps_api.patch_namespaced_deployment(
namespace=self.__namespace,
body=self.load_gen_deployment,
name=get_name(self.load_gen_deployment),
async_req=True,
))
def create(self):
"""Creates the Postgres resources in the cluster
"""
# Check to see if this isn't meant to deploy first
if not self.__config["deploy"]:
return
evaluate_request(
self.__v1_api.create_namespaced_service(
namespace=self.__namespace, body=self.postgres_service, async_req=True
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_api.create_namespaced_config_map(
namespace=self.__namespace,
body=self.postgres_config_map,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_api.create_namespaced_persistent_volume_claim(
namespace=self.__namespace, body=self.postgres_pvc, async_req=True
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace, body=self.postgres, async_req=True
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace, body=self.postgres_connector, async_req=True
),
allowed_statuses=[409],
)
def delete(self):
"""Removes the Postgres resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.postgres_service),
async_req=True,
)
)
evaluate_request(
self.__v1_api.delete_namespaced_config_map(
namespace=self.__namespace,
name=get_name(self.postgres_config_map),
async_req=True,
)
)
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.postgres_pvc),
async_req=True,
)
)
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace, name=get_name(self.postgres), async_req=True
)
)
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.postgres_connector),
async_req=True,
)
)
def create(self):
"""Creates the Elasticsearch resources in the cluster
"""
# Check to see if this isn't meant to deploy first
if not self.config["deploy"]:
return
evaluate_request(
self.v1_api.create_namespaced_config_map(
namespace=self.namespace,
body=self.elasticsearch_config_map,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.v1_apps_api.create_namespaced_deployment(
namespace=self.namespace,
body=self.elasticsearch_master,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.v1_api.create_namespaced_service(
namespace=self.namespace,
body=self.elasticsearch_discovery,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.v1_apps_api.create_namespaced_stateful_set(
namespace=self.namespace,
body=self.elasticsearch_data,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.v1_api.create_namespaced_service(
namespace=self.namespace,
body=self.elasticsearch_data_service,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.v1_apps_api.create_namespaced_deployment(
namespace=self.namespace,
body=self.elasticsearch_client,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.v1_api.create_namespaced_service(
namespace=self.namespace,
body=self.elasticsearch_client_service,
async_req=True,
),
allowed_statuses=[409],
)
self.logstash_manager.create()
def delete(self):
"""Removes the Elasticsearch resources from the cluster
"""
evaluate_request(
self.v1_api.delete_namespaced_config_map(
namespace=self.namespace,
name=get_name(self.elasticsearch_config_map),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_deployment(
namespace=self.namespace,
name=get_name(self.elasticsearch_master),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_discovery),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_stateful_set(
namespace=self.namespace,
name=get_name(self.elasticsearch_data),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_data_service),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_deployment(
namespace=self.namespace,
name=get_name(self.elasticsearch_client),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_client_service),
async_req=True,
))
self.logstash_manager.delete()
def delete(self):
"""Remove the Redis resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_pod(
namespace=self.__namespace,
name=get_name(self.redis_master),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.redis_service),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_replication_controller(
namespace=self.__namespace,
name=get_name(self.redis_slave_controller),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_replication_controller(
namespace=self.__namespace,
name=get_name(self.redis_sentinel_controller),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.redis_metrics_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.redis_connector),
async_req=True,
propagation_policy="Background",
))
def create(self):
"""Create the Redis resources in the cluster
"""
# Check to see if this isn't meant to deploy first
if not self.__config["deploy"]:
return
redis_pods = self.__v1_api.list_namespaced_pod(
namespace=self.__namespace, label_selector="name=redis")
if not len(redis_pods.items) == 0:
return
evaluate_request(
self.__v1_api.create_namespaced_pod(namespace=self.__namespace,
body=self.redis_master,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_api.create_namespaced_service(namespace=self.__namespace,
body=self.redis_service,
async_req=True),
allowed_statuses=[409],
)
self.__wait_for_redis_master()
evaluate_request(
self.__v1_api.create_namespaced_replication_controller(
namespace=self.__namespace,
body=self.redis_slave_controller,
async_req=True,
))
evaluate_request(
self.__v1_api.create_namespaced_replication_controller(
namespace=self.__namespace,
body=self.redis_sentinel_controller,
async_req=True,
))
self.__wait_for_redis_slaves()
self.__v1_api.delete_namespaced_pod(namespace=self.__namespace,
name=get_name(self.redis_master))
evaluate_request(
self.__v1_api.create_namespaced_service(
namespace=self.__namespace,
body=self.redis_metrics_service,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace,
body=self.redis_connector,
async_req=True),
allowed_statuses=[409],
)
def create(self):
"""Create kafka items in the cluster.
Creates all the parts of the kafka deployment in the cluster. If items already exist in
the cluster, those items will be quietly ignored.
"""
if self.__config["usePersistentVolume"]:
evaluate_request(
self.__v1_api.create_namespaced_persistent_volume_claim(
namespace=self.__namespace,
body=self.kafka_pvc,
async_req=True),
allowed_statuses=[409],
)
else:
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.kafka_pvc),
async_req=True,
))
evaluate_request(
self.__v1_api.create_namespaced_service(namespace=self.__namespace,
body=self.kafka_service,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_policy_api.create_namespaced_pod_disruption_budget(
namespace=self.__namespace,
body=self.kafka_pdb,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_stateful_set(
namespace=self.__namespace, body=self.kafka, async_req=True),
allowed_statuses=[409],
)
# Wait for Kafka to register itself with zookeeper before proceeding
kafka_started = False
while not kafka_started:
try:
out = stream(
self.__v1_api.connect_get_namespaced_pod_exec,
"zk-0",
self.__namespace,
command=[
"bash",
"-c",
"echo dump | nc localhost 2181 | grep brokers",
],
stderr=False,
stdin=False,
stdout=True,
tty=False,
)
kafka_started = len(out.split()) == 3
except Exception as _:
sleep(2)
evaluate_request(
self.__v1_api.create_namespaced_service(
namespace=self.__namespace,
body=self.kafka_metrics_service,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace,
body=self.kafka_metrics,
async_req=True),
allowed_statuses=[409],
)
def delete(self):
"""Delete all Kafka artifacts within the cluster
Removes all the Kafka artifacts within the cluster safely - if something does not already
exist within the cluster, it will be ignored.
"""
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.kafka_service),
async_req=True,
))
evaluate_request(
self.__v1_policy_api.delete_namespaced_pod_disruption_budget(
namespace=self.__namespace,
name=get_name(self.kafka_pdb),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_stateful_set(
namespace=self.__namespace,
name=get_name(self.kafka),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.kafka_metrics_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.kafka_metrics),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.kafka_pvc),
async_req=True,
))