def setup_clean(self, username, table_name):
"""
创建清洗配置
"""
params = {
"raw_data_id": self.raw_data_id,
"json_config": self.json_config,
"pe_config": "",
"bk_biz_id": self.project_data.cc_biz_id,
"result_table_name": table_name,
"result_table_name_alias": table_name,
"fields": self.clean_fields,
"clean_config_name": f"clean_{self.data_type}_config",
"description": f"clean_{self.data_type}",
}
params.update(compose_auth_params(username))
resp = http_post(f"{DATA_API_V3_PREFIX}/databus/cleans/", json=params)
if resp.get("result"):
return True, resp.get("data", {}).get("id")
return False, resp.get("message")
def create_namespace(access_token,
project_id,
cluster_id,
name,
description,
creator,
env_type,
has_image_secret=None):
"""创建namespace
"""
url = f"{BCS_CC_API_PRE_URL}/projects/{project_id}/clusters/{cluster_id}/namespaces/"
params = {"access_token": access_token}
payload = {
"name": name,
"description": description,
"creator": creator,
"env_type": env_type
}
if has_image_secret is not None:
payload["has_image_secret"] = has_image_secret
return http_post(url, params=params, json=payload)
def create_namespace(access_token,
project_id,
cluster_id,
name,
description,
creator,
env_type,
has_image_secret=None):
"""创建namespace
"""
url = f'{CC_HOST}/projects/{project_id}/clusters/{cluster_id}/namespaces/'
params = {'access_token': access_token}
payload = {
'name': name,
'description': description,
'creator': creator,
'env_type': env_type
}
if has_image_secret is not None:
payload['has_image_secret'] = has_image_secret
return http_post(url, params=params, json=payload)
def setup_clean(bk_username, bk_biz_id, raw_data_id, data_type):
"""
创建清洗配置,并启动清洗任务
"""
url = f'{DATA_API_V3_PREFIX}/databus/scenarios/setup_clean/'
# result_table_name: 清洗配置英文标识。英文标识在业务下唯一，重复创建会报错
# 唯一,小于15字符,符合正则'^[a-zA-Z][a-zA-Z0-9_]*$'
result_table_name = '%s_%s' % (data_type[:4], int(time.time()))
if data_type == DataType.SLOG.value:
fields = SLOG_CLEAN_FIELDS
json_config = SLOG_JSON_CONDIF
description = "标准日志采集清洗任务"
elif data_type == DataType.CLOG.value:
fields = CLOG_CLEAN_FIELDS
json_config = CLOG_JSON_CONDIF
description = "非标准日志采集清洗任务"
elif data_type == DataType.METRIC.value:
fields = METRIC_CLEAN_FIELDS
json_config = METRIC_JSON_CONDIF
description = "BCS metric清洗任务"
data = {
"data_token": DATA_TOKEN,
"raw_data_id": raw_data_id,
"result_table_name": result_table_name,
"result_table_name_alias": result_table_name,
"bk_biz_id": bk_biz_id,
"bk_username": bk_username,
"description": description,
"fields": fields,
"json_config": json_config,
"pe_config": "",
}
resp = http_post(url, json=data)
if resp.get('result'):
return True, resp.get('data', {}).get('result_table_id')
return False, resp.get('message')
def deploy_plan(bk_username, bk_biz_id, data_name, data_type):
"""
提交接入部署计划,获取dataid
data_type: slog/clog/metric
data_name: data_name/dataset(create_metric_data_id)
'^[a-zA-Z][a-zA-Z0-9_]*$'，业务下唯一，长度限制了 15 个字符
"""
description = ''
if data_type == DataType.SLOG.value:
description = "标准日志采集"
elif data_type == DataType.CLOG.value:
description = "非标准日志采集"
elif data_type == DataType.METRIC.value:
description = 'BCS metric'
data = {
'bk_app_code': settings.APP_ID,
'bk_app_secret': settings.APP_TOKEN,
'bk_username': bk_username,
'bkdata_authentication_method': 'user',
'data_scenario': 'log',
'bk_biz_id': bk_biz_id,
'description': description,
'data_token': DATA_TOKEN,
"access_raw_data": {
'raw_data_alias': data_name,
'raw_data_name': data_name,
'maintainer': bk_username,
'description': description,
'data_category': 'online',
'data_source': 'svr',
'data_encoding': 'UTF-8',
'sensitivity': 'private'
},
}
resp = http_post(f'{DATA_API_V3_PREFIX}/access/deploy_plan/', json=data)
if resp.get('result'):
return True, resp.get('data', {}).get('raw_data_id')
return False, resp.get('message')
def get_multi_perm_resource(self, user_id, res_list):
"""
批量查询用户有某个权限的资源
res_list:[
{
"action_id": "read",
"resource_type": "namespace",
},
{
"action_id": "manage",
"resource_type": "namespace"
}
]
"""
url = f'{settings.BK_IAM_HOST}/{self.system_prefix}/{self.system_id}/authorized-resources/search'
data = {
"scope_type": "project",
"scope_id": self.project_code,
"principal_type": "user",
"principal_id": user_id,
"resource_types_actions": res_list
}
return http_post(url, json=data, **self.kwargs)
def get_mesos_app_taskgroup(self,
taskgroup_name=None,
namespace=None,
app_name=None,
field=None):
"""获取taskgroup"""
url = "{host}/query/mesos/dynamic/clusters/{cluster_id}/taskgroup".format(
host=self.storage_host, cluster_id=self.cluster_id)
# 可选参数
params = {}
if taskgroup_name:
params["name"] = taskgroup_name
if namespace:
params["namespace"] = namespace
if app_name:
params["rcName"] = app_name
if field:
params["field"] = field
kwargs = {"headers": self.headers}
resp = http_post(url, json=params, **kwargs)
return resp
def get_access_token(params):
url = f"{settings.BK_SSM_HOST}/api/v1/auth/access-tokens"
return utils.http_post(url, json=params, headers=HEADERS)
def create_service_monitor(self, namespace, spec):
"""创建servicemonitor"""
# Mesos API Version 是BCS定制
spec["apiVersion"] = SERVICE_MONITOR_API_VERSION
url = self._get_service_monitor_url(namespace)
return http_post(url, json=spec, headers=self.headers, raise_for_status=False)
def get_statefulset_with_post(self, data):
"""通过post方法，查询statefulset
"""
url = '{host}/query/k8s/dynamic/clusters/{cluster_id}/statefulset'.format(
host=self.storage_host, cluster_id=self.cluster_id)
return http_post(url, json=data, headers=self.headers)
def get_cluster_list(access_token, project_id, cluster_ids):
url = f"{BCS_CC_API_PRE_URL}/projects/{project_id}/clusters_list/"
params = {"access_token": access_token}
data = {"cluster_ids": cluster_ids} if cluster_ids else None
return http_post(url, params=params, json=data)
def create_service_monitor(self, namespace, spec):
url = f"{self.context['host']}/apis/monitoring.coreos.com/v1/namespaces/{namespace}/servicemonitors"
return http_post(url, json=spec, headers=self._headers_for_service_monitor, raise_for_status=False)
def get_cluster_list(access_token, project_id, cluster_ids):
url = f'{CC_HOST}/projects/{project_id}/clusters_list/'
params = {'access_token': access_token}
data = {'cluster_ids': cluster_ids} if cluster_ids else None
return http_post(url, params=params, json=data)
def update_agent_attrs(self, attrs):
"""批量修改/增加节点属性
"""
url = "{host}/mesos/agentsettings".format(host=self.scheduler_host)
return http_post(url, json=attrs, headers=self.headers)
def create_deployment(self, namespace, data):
url = "{host}/mesos/namespaces/{ns}/deployments".format(
host=self.scheduler_host, ns=namespace)
resp = http_post(url, json=data, headers=self.headers)
return resp
def create_application(self, namespace, data):
"""创建application"""
url = "{host}/mesos/namespaces/{ns}/applications".format(host=self.scheduler_host, ns=namespace)
kwargs = {"headers": self.headers}
resp = http_post(url, json=data, **kwargs)
return resp
def get_authorization_by_access_token(access_token):
url = f"{settings.BK_SSM_HOST}/api/v1/auth/access-tokens/verify"
return utils.http_post(url,
json={"access_token": access_token},
headers=HEADERS)
def send_application_signal(self, namespace, name, data):
"""对指定的application下所有的running状态的taskgroup发送信息
"""
url = "{host}/mesos/namespaces/{namespace}/applications/{name}/message".format(
host=self.scheduler_host, namespace=namespace, name=name)
return http_post(url, json=data, headers=self.headers)
def create_hpa(self, namespace, spec):
"""创建HPA
"""
url = f'{self.api_host}/v4/scheduler/mesos/crd/namespaces/{namespace}/autoscaler'
result = http_post(url, json=spec, headers=self.headers)
return result
def apply_cidr(self, ip_number: int, vpc: str) -> dict:
url = f"{self.rest_host}/cidr/apply_cidr"
params = {"access_token": self.access_token}
data = {"ip_number": ip_number, "cluster": self.cluster_id, "vpc": vpc}
return http_post(url, params=params, json=data, raise_for_status=False)
def create_service_monitor(self, namespace, spec):
context = self.get_context_or_raise()
host = f"{self._bcs_server_host}{context['server_address_path']}".rstrip("/")
headers = {"Authorization": f"Bearer {context['user_token']}"}
url = f"{host}/apis/monitoring.coreos.com/v1/namespaces/{namespace}/servicemonitors"
return http_post(url, json=spec, headers=headers, raise_for_status=False)
def create_project(access_token, data):
url = f"{BCS_CC_API_PRE_URL}/projects/"
return http_post(url, params={"access_token": access_token}, json=data)
def enable_agent(self, ip):
"""启用Agent"""
url = "{host}/mesos/agentsettings/enable".format(host=self.scheduler_host)
params = {"ips": ip}
resp = http_post(url, params=params, headers=self.headers)
return resp
def create_namespace(self, data):
"""创建namespaces
"""
url = '{host}/namespaces'.format(host=self.scheduler_host)
resp = http_post(url, json=data, headers=self.headers)
return resp
def get_client_access_token(self):
"""获取非用户态access_token
"""
url = f'{settings.BK_IAM_HOST}/bkiam/api/v1/auth/access-tokens'
data = {'grant_type': 'client_credentials', 'id_provider': 'client'}
return http_post(url, json=data, **self.kwargs)
def create_clusterrolebindings(self, namespace, data):
"""创建 ClusterRoleBinding
"""
url = '{host}/clusterrolebindings'.format(host=self.scheduler_host)
resp = http_post(url, json=data, headers=self.headers, **HTTP_KWARGS)
return resp
def send_command(self, category, namespace, name, config):
"""发送命令"""
url = "{host}/mesos/command/{category}/{ns}/{name}".format(
host=self.scheduler_host, category=category, ns=namespace, name=name
)
return http_post(url, json=config, headers=self.headers)