def hpa_apply():
try:
reload(MyForm)
form = MyForm.FormK8sHpa()
if form.submit.data:
context = form.contexts.data
deployment = form.deployment.data
max_replica = form.max_replica.data
min_replica = form.min_replica.data
cpu_value = form.cpu_value.data
if max_replica and min_replica and cpu_value:
exist_hpa = []
config.load_kube_config(config_file, context)
api_instance = client.AutoscalingV1Api()
try:
ret = api_instance.list_horizontal_pod_autoscaler_for_all_namespaces(
)
for i in ret.items:
exist_hpa.append(i.spec.scale_target_ref.name)
if deployment in exist_hpa:
#配置已存在进行更新
api_instance.patch_namespaced_horizontal_pod_autoscaler(
name='%s-hpa' % deployment,
namespace=namespace,
body=client.V1HorizontalPodAutoscaler(
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=int(max_replica),
min_replicas=int(min_replica),
target_cpu_utilization_percentage=int(
cpu_value),
scale_target_ref=client.
V1CrossVersionObjectReference(
api_version='extensions/v1beta1',
kind='Deployment',
name=deployment))))
else:
# 配置不存在进行创建
api_instance.create_namespaced_horizontal_pod_autoscaler(
namespace=namespace,
body=client.V1HorizontalPodAutoscaler(
metadata=client.V1ObjectMeta(
name='%s-hpa' % deployment,
namespace=namespace),
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=int(max_replica),
min_replicas=int(min_replica),
target_cpu_utilization_percentage=int(
cpu_value),
scale_target_ref=client.
V1CrossVersionObjectReference(
api_version='extensions/v1beta1',
kind='Deployment',
name=deployment))))
return redirect(url_for('k8s.hpa'))
except Exception as e:
logging.error(e)
except Exception as e:
logging.error(e)
return redirect(url_for('error'))
return render_template('k8s_hpa.html', form=form)
def init_hpa(self, tosca_kube_obj, kube_obj_name):
scaling_props = tosca_kube_obj.scaling_object
hpa = None
if scaling_props:
min_replicas = scaling_props.min_replicas
max_replicas = scaling_props.max_replicas
cpu_util = scaling_props.target_cpu_utilization_percentage
deployment_name = kube_obj_name
# Create target Deployment object
target = client.V1CrossVersionObjectReference(
api_version="extensions/v1beta1",
kind="Deployment",
name=deployment_name)
# Create the specification of horizon pod auto-scaling
hpa_spec = client.V1HorizontalPodAutoscalerSpec(
min_replicas=min_replicas,
max_replicas=max_replicas,
target_cpu_utilization_percentage=cpu_util,
scale_target_ref=target)
metadata = client.V1ObjectMeta(name=deployment_name)
# Create Horizon Pod Auto-Scaling
hpa = client.V1HorizontalPodAutoscaler(api_version="autoscaling/v1",
kind="HorizontalPodAutoscaler",
spec=hpa_spec,
metadata=metadata)
return hpa
def modify_k8s_hpa():
infos = None
status = None
try:
Infos = request.get_json()
api_instance = client.AutoscalingV1Api()
if request.method == 'POST':
try:
api_instance.patch_namespaced_horizontal_pod_autoscaler(name=Infos['name'], namespace=namespace,
body=client.V1HorizontalPodAutoscaler(
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=int(Infos['max_update']),
target_cpu_utilization_percentage=int(Infos['cpu_update'].replace('%','')),
scale_target_ref=client.V1CrossVersionObjectReference(
kind='Deployment',
name=Infos['target_ref']))))
except Exception as e:
logging.error(e)
infos = '修改参数失败!'
else:
status = 'ok'
infos = '修改参数成功!'
if request.method == 'DELETE':
try:
api_instance.delete_namespaced_horizontal_pod_autoscaler(name=Infos['name'], namespace=namespace,body=client.V1DeleteOptions())
except Exception as e:
logging.error(e)
infos = '删除%s失败!' %Infos['name']
else:
status = 'ok'
infos = '删除%s成功!' %Infos['name']
except Exception as e:
logging.error(e)
finally:
return jsonify({'status':status,'infos':infos})
def addHPA(rev_data):
optDict = {}
for i in rev_data.split('&'):
temp = i.split('=')
opt = {temp[0]: temp[1].strip()}
optDict.update(opt)
# print(optDict)
api_instance = client.AutoscalingV1Api()
hpa_target_ref = client.V1CrossVersionObjectReference(name=optDict['current_hpa_deployment'], kind=kind)
hpa_spec = client.V1HorizontalPodAutoscalerSpec(max_replicas=optDict['hpa_max_qant'],
scale_target_ref=hpa_target_ref,
min_replicas=optDict['hpa_min_qant'],
target_cpu_utilization_percentage=optDict['cpu_threshold'])
hpa_metadata = client.V1ObjectMeta(name=optDict['current_hpa_deployment'], namespace=optDict['current_ns'])
hpa_status = client.V1HorizontalPodAutoscalerStatus()
# hpa_metadata = client.V1ObjectMeta(name=optDict['current_hpa_deployment'], namespace=optDict['current_ns'])
body = client.V1HorizontalPodAutoscaler(metadata=hpa_metadata, spec=hpa_spec)
namespace = optDict['current_ns'] # str | object name and auth scope, such as for teams and projects
pretty = 'true' # str | If 'true', then the output is pretty printed. (optional)
api_status = {}
try:
api_response = api_instance.create_namespaced_horizontal_pod_autoscaler(namespace=namespace, body=body,
pretty=pretty)
pprint(api_response)
api_status['current_replicas'] = hpa_status.current_replicas
api_status['cpu_util'] = hpa_status.current_cpu_utilization_percentage
api_status['last_scale_time'] = hpa_status.last_scale_time
api_status['status_code'] = 200
return api_status
except ApiException as e:
print("Exception when calling AutoscalingV1Api->create_namespaced_horizontal_pod_autoscaler: %s\n" % e)
def horizontal_scale(self, name, pod_name, pod_kind, max, metric):
"""Create horizontal pod scaller, based on metric."""
api = client.AutoscalingV1Api()
pd_scale = client.V1HorizontalPodAutoscaler()
pd_scale.metadata = client.V1ObjectMeta(name=name)
target = client.V1CrossVersionObjectReference(name=pod_name, kind=pod_kind)
spec = client.V1HorizontalPodAutoscalerSpec(min_replicas=1, max_replicas=max, scale_target_ref=target)
status = client.V1HorizontalPodAutoscalerStatus(current_replicas=1, desired_replicas=2)
pd_scale.spec = spec
pd_scale.status = status
try:
api.create_namespaced_horizontal_pod_autoscaler(namespace=self._namespace, body=pd_scale)
LOG.info(f'Persistent Volume: {name} created.')
except ApiException as e:
LOG.error(f'Exception message: {e}')
def create_nlu_hpa_object(hpa_name):
deployment = client.V1CrossVersionObjectReference(
api_version="extensions/v1beta1",
kind="Deployment",
name=hpa_name)
spec = client.V1HorizontalPodAutoscalerSpec(
max_replicas=5,
min_replicas=2,
scale_target_ref=deployment,
target_cpu_utilization_percentage=100)
hpa = client.V1HorizontalPodAutoscaler(
api_version="autoscaling/v1",
kind="HorizontalPodAutoscaler",
metadata=client.V1ObjectMeta(name=hpa_name),
spec=spec)
return hpa
def create_hpa_object(self):
spec = client.V1HorizontalPodAutoscalerSpec(
min_replicas=self.min_replicas,
max_replicas=self.max_replicas,
target_cpu_utilization_percentage=self.
target_cpu_utilization_percentage,
scale_target_ref=client.V1CrossVersionObjectReference(
api_version=self.target_api_version,
kind=self.target_kind,
name=self.target_name))
hpa = client.V1HorizontalPodAutoscaler(
api_version='autoscaling/v1',
kind='HorizontalPodAutoscaler',
metadata=client.V1ObjectMeta(name=self.hpa_name),
spec=spec)
return hpa
def hpa_definition(self):
return client.V1HorizontalPodAutoscaler(
metadata=client.V1ObjectMeta(
name=self.kube_name,
namespace=settings.kube_namespace,
),
spec=client.V1HorizontalPodAutoscalerSpec(
min_replicas=1,
max_replicas=100,
scale_target_ref=client.V1CrossVersionObjectReference(
kind="MicroserviceCluster",
name=self.kube_name,
),
target_cpu_utilization_percentage=50,
),
)
def create_hpa_object(request_id, workers):
target = client.V1CrossVersionObjectReference(api_version="apps/v1",
kind='Deployment',
name="transformer-" +
request_id)
hpa = client.V1HorizontalPodAutoscaler(
api_version="autoscaling/v1",
kind='HorizontalPodAutoscaler',
metadata=client.V1ObjectMeta(name="transformer-" + request_id),
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=workers,
scale_target_ref=target,
target_cpu_utilization_percentage=current_app.
config['TRANSFORMER_CPU_SCALE_THRESHOLD']))
return hpa
def apply_rekcurd_to_kubernetes(project_id: int,
application_id: str,
service_level: str,
version: str,
insecure_host: str,
insecure_port: int,
replicas_default: int,
replicas_minimum: int,
replicas_maximum: int,
autoscale_cpu_threshold: str,
policy_max_surge: int,
policy_max_unavailable: int,
policy_wait_seconds: int,
container_image: str,
resource_request_cpu: str,
resource_request_memory: str,
resource_limit_cpu: str,
resource_limit_memory: str,
commit_message: str,
service_model_assignment: int,
service_git_url: str = "",
service_git_branch: str = "",
service_boot_script: str = "",
debug_mode: bool = False,
service_id: str = None,
is_creation_mode: bool = False,
display_name: str = None,
description: str = None,
kubernetes_models=None,
**kwargs) -> str:
"""
kubectl apply
:param project_id:
:param application_id:
:param service_level:
:param version:
:param insecure_host:
:param insecure_port:
:param replicas_default:
:param replicas_minimum:
:param replicas_maximum:
:param autoscale_cpu_threshold:
:param policy_max_surge:
:param policy_max_unavailable:
:param policy_wait_seconds:
:param container_image:
:param resource_request_cpu:
:param resource_request_memory:
:param resource_limit_cpu:
:param resource_limit_memory:
:param commit_message:
:param service_model_assignment:
:param service_git_url:
:param service_git_branch:
:param service_boot_script:
:param debug_mode:
:param service_id:
:param is_creation_mode:
:param display_name:
:param description:
:param kubernetes_models:
:param kwargs:
:return:
"""
__num_retry = 5
progress_deadline_seconds = \
int(__num_retry*policy_wait_seconds*replicas_maximum/(policy_max_surge+policy_max_unavailable))
if service_id is None:
is_creation_mode = True
service_id = uuid.uuid4().hex
if kubernetes_models is None:
kubernetes_models = db.session.query(KubernetesModel).filter(
KubernetesModel.project_id == project_id).all()
data_server_model: DataServerModel = db.session.query(
DataServerModel).filter(
DataServerModel.project_id == project_id).first_or_404()
application_model: ApplicationModel = db.session.query(
ApplicationModel).filter(
ApplicationModel.application_id == application_id).first_or_404()
application_name = application_model.application_name
model_model: ModelModel = db.session.query(ModelModel).filter(
ModelModel.model_id == service_model_assignment).first_or_404()
from kubernetes import client
try:
git_secret = load_secret(project_id, application_id, service_level,
GIT_SECRET_PREFIX)
except:
git_secret = None
volume_mounts = dict()
volumes = dict()
if git_secret:
connector_name = "sec-git-name"
secret_name = "sec-{}-{}".format(GIT_SECRET_PREFIX, application_id)
volume_mounts = {
'volume_mounts': [
client.V1VolumeMount(name=connector_name,
mount_path=GIT_SSH_MOUNT_DIR,
read_only=True)
]
}
volumes = {
'volumes': [
client.V1Volume(name=connector_name,
secret=client.V1SecretVolumeSource(
secret_name=secret_name,
items=[
client.V1KeyToPath(key=GIT_ID_RSA,
path=GIT_ID_RSA,
mode=GIT_SSH_MODE),
client.V1KeyToPath(key=GIT_CONFIG,
path=GIT_CONFIG,
mode=GIT_SSH_MODE)
]))
]
}
for kubernetes_model in kubernetes_models:
full_config_path = get_full_config_path(kubernetes_model.config_path)
from kubernetes import config
config.load_kube_config(full_config_path)
pod_env = [
client.V1EnvVar(name="REKCURD_SERVICE_UPDATE_FLAG",
value=commit_message),
client.V1EnvVar(name="REKCURD_KUBERNETES_MODE", value="True"),
client.V1EnvVar(name="REKCURD_DEBUG_MODE", value=str(debug_mode)),
client.V1EnvVar(name="REKCURD_APPLICATION_NAME",
value=application_name),
client.V1EnvVar(name="REKCURD_SERVICE_INSECURE_HOST",
value=insecure_host),
client.V1EnvVar(name="REKCURD_SERVICE_INSECURE_PORT",
value=str(insecure_port)),
client.V1EnvVar(name="REKCURD_SERVICE_ID", value=service_id),
client.V1EnvVar(name="REKCURD_SERVICE_LEVEL", value=service_level),
client.V1EnvVar(name="REKCURD_GRPC_PROTO_VERSION", value=version),
client.V1EnvVar(name="REKCURD_MODEL_MODE",
value=data_server_model.data_server_mode.value),
client.V1EnvVar(name="REKCURD_MODEL_FILE_PATH",
value=model_model.filepath),
client.V1EnvVar(name="REKCURD_CEPH_ACCESS_KEY",
value=str(data_server_model.ceph_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_SECRET_KEY",
value=str(data_server_model.ceph_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_HOST",
value=str(data_server_model.ceph_host or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_PORT",
value=str(data_server_model.ceph_port or "1234")),
client.V1EnvVar(name="REKCURD_CEPH_IS_SECURE",
value=str(data_server_model.ceph_is_secure
or "False")),
client.V1EnvVar(name="REKCURD_CEPH_BUCKET_NAME",
value=str(data_server_model.ceph_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_ACCESS_KEY",
value=str(data_server_model.aws_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_SECRET_KEY",
value=str(data_server_model.aws_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_BUCKET_NAME",
value=str(data_server_model.aws_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_ACCESS_KEY",
value=str(data_server_model.gcs_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_SECRET_KEY",
value=str(data_server_model.gcs_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_BUCKET_NAME",
value=str(data_server_model.gcs_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_SERVICE_GIT_URL",
value=service_git_url),
client.V1EnvVar(name="REKCURD_SERVICE_GIT_BRANCH",
value=service_git_branch),
client.V1EnvVar(name="REKCURD_SERVICE_BOOT_SHELL",
value=service_boot_script),
]
"""Namespace registration."""
core_vi_api = client.CoreV1Api()
try:
core_vi_api.read_namespace(name=service_level)
except:
api.logger.info("\"{}\" namespace created".format(service_level))
v1_namespace = client.V1Namespace(
api_version="v1",
kind="Namespace",
metadata=client.V1ObjectMeta(name=service_level))
core_vi_api.create_namespace(body=v1_namespace)
"""Create/patch Deployment."""
v1_deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name="deploy-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1DeploymentSpec(
min_ready_seconds=policy_wait_seconds,
progress_deadline_seconds=progress_deadline_seconds,
replicas=replicas_default,
revision_history_limit=3,
selector=client.V1LabelSelector(
match_labels={"sel": service_id}),
strategy=client.V1DeploymentStrategy(
type="RollingUpdate",
rolling_update=client.V1RollingUpdateDeployment(
max_surge=policy_max_surge,
max_unavailable=policy_max_unavailable)),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1PodSpec(affinity=client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution
=[
client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=client.
V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key="id",
operator="In",
values=[service_id])
]),
topology_key="kubernetes.io/hostname"),
weight=100)
])),
containers=[
client.V1Container(
env=pod_env,
image=container_image,
image_pull_policy="Always",
name=service_id,
ports=[
client.V1ContainerPort(
container_port=
insecure_port)
],
resources=client.
V1ResourceRequirements(
limits={
"cpu":
str(resource_limit_cpu
),
"memory":
resource_limit_memory
},
requests={
"cpu":
str(resource_request_cpu
),
"memory":
resource_request_memory
}),
security_context=client.
V1SecurityContext(
privileged=True),
**volume_mounts)
],
node_selector={
"host": service_level
},
**volumes))))
apps_v1_api = client.AppsV1Api()
if is_creation_mode:
api.logger.info("Deployment created.")
apps_v1_api.create_namespaced_deployment(body=v1_deployment,
namespace=service_level)
else:
api.logger.info("Deployment patched.")
apps_v1_api.patch_namespaced_deployment(
body=v1_deployment,
name="deploy-{0}".format(service_id),
namespace=service_level)
"""Create/patch Service."""
v1_service = client.V1Service(
api_version="v1",
kind="Service",
metadata=client.V1ObjectMeta(name="svc-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1ServiceSpec(ports=[
client.V1ServicePort(name="grpc-backend",
port=insecure_port,
protocol="TCP",
target_port=insecure_port)
],
selector={"sel": service_id}))
core_vi_api = client.CoreV1Api()
if is_creation_mode:
api.logger.info("Service created.")
core_vi_api.create_namespaced_service(namespace=service_level,
body=v1_service)
else:
api.logger.info("Service patched.")
core_vi_api.patch_namespaced_service(
namespace=service_level,
name="svc-{0}".format(service_id),
body=v1_service)
"""Create/patch Autoscaler."""
v1_horizontal_pod_autoscaler = client.V1HorizontalPodAutoscaler(
api_version="autoscaling/v1",
kind="HorizontalPodAutoscaler",
metadata=client.V1ObjectMeta(name="hpa-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=replicas_maximum,
min_replicas=replicas_minimum,
scale_target_ref=client.V1CrossVersionObjectReference(
api_version="apps/v1",
kind="Deployment",
name="deploy-{0}".format(service_id)),
target_cpu_utilization_percentage=autoscale_cpu_threshold))
autoscaling_v1_api = client.AutoscalingV1Api()
if is_creation_mode:
api.logger.info("Autoscaler created.")
autoscaling_v1_api.create_namespaced_horizontal_pod_autoscaler(
namespace=service_level, body=v1_horizontal_pod_autoscaler)
else:
api.logger.info("Autoscaler patched.")
autoscaling_v1_api.patch_namespaced_horizontal_pod_autoscaler(
namespace=service_level,
name="hpa-{0}".format(service_id),
body=v1_horizontal_pod_autoscaler)
"""Create Istio ingress if this is the first application."""
custom_object_api = client.CustomObjectsApi()
try:
custom_object_api.get_namespaced_custom_object(
group="networking.istio.io",
version="v1alpha3",
namespace=service_level,
plural="virtualservices",
name="ing-vs-{0}".format(application_id),
)
except:
ingress_virtual_service_body = {
"apiVersion": "networking.istio.io/v1alpha3",
"kind": "VirtualService",
"metadata": {
"labels": {
"rekcurd-worker": "True",
"id": application_id,
"name": application_name
},
"name": "ing-vs-{0}".format(application_id),
"namespace": service_level
},
"spec": {
"hosts": ["*"],
"gateways": ["rekcurd-ingress-gateway"],
"http": [{
"match": [{
"headers": {
"x-rekcurd-application-name": {
"exact": application_name
},
"x-rekcurd-sevice-level": {
"exact": service_level
},
"x-rekcurd-grpc-version": {
"exact": version
},
}
}],
"route": [{
"destination": {
"port": {
"number": insecure_port
},
"host": "svc-{0}".format(service_id)
},
"weight": 100
}],
"retries": {
"attempts": 25,
"perTryTimeout": "1s"
}
}]
}
}
api.logger.info("Istio created.")
custom_object_api.create_namespaced_custom_object(
group="networking.istio.io",
version="v1alpha3",
namespace=service_level,
plural="virtualservices",
body=ingress_virtual_service_body)
"""Add service model."""
if is_creation_mode:
if display_name is None:
display_name = "{0}-{1}".format(service_level, service_id)
service_model = ServiceModel(service_id=service_id,
application_id=application_id,
display_name=display_name,
description=description,
service_level=service_level,
version=version,
model_id=service_model_assignment,
insecure_host=insecure_host,
insecure_port=insecure_port)
db.session.add(service_model)
db.session.flush()
"""Finish."""
return service_id
