def file_upload(request, project_id):
"""
@summary: 本地文件上传
@param request:
@param project_id:
@return:
"""
file_manager_type = EnvironmentVariables.objects.get_var(
'BKAPP_FILE_MANAGER_TYPE')
if not file_manager_type:
return JsonResponse({
'result': False,
'message': u"File Manager 未配置，请联系管理员进行配置"
})
try:
file_manager = ManagerFactory.get_manager(
manager_type=file_manager_type)
except Exception as e:
logger.error('can not get file manager for type: {}\n err: {}'.format(
file_manager_type, traceback.format_exc()))
return JsonResponse({'result': False, 'message': str(e)})
file_obj = request.FILES['file']
file_name = file_obj.name
file_size = file_obj.size
# 文件名不能包含中文， 文件大小不能大于 2G
if file_size > 2048 * 1024 * 1024:
message = _(u"文件上传失败， 文件大小超过2G")
response = JsonResponse({'result': False, 'message': message})
response.status_code = 400
return response
if INVALID_CHAR_REGEX.findall(file_name):
message = _(u"文件上传失败，文件名不能包含中文和\\/:*?\"<>|等特殊字符")
response = JsonResponse({'result': False, 'message': message})
response.status_code = 400
return response
shims = 'plugins_upload/job_push_local_files/{}'.format(project_id)
try:
file_tag = file_manager.save(name=file_name,
content=file_obj,
shims=shims)
except Exception:
logger.error('file upload save err: {}'.format(traceback.format_exc()))
return JsonResponse({
'result': False,
'message': _(u"文件上传归档失败，请联系管理员")
})
return JsonResponse({'result': True, 'tag': file_tag})
def execute(self, data, parent_data):
executor = parent_data.inputs.executor
biz_cc_id = data.inputs.biz_cc_id
local_files = data.inputs.job_local_files
target_ip_list = data.inputs.job_target_ip_list
target_account = data.inputs.job_target_account
target_path = data.inputs.job_target_path
file_manager_type = EnvironmentVariables.objects.get_var(
'BKAPP_FILE_MANAGER_TYPE')
if not file_manager_type:
data.outputs.ex_data = 'File Manager configuration error, contact administrator please.'
return False
try:
file_manager = ManagerFactory.get_manager(
manager_type=file_manager_type)
except Exception as e:
err_msg = 'can not get file manager for type: {}\n err: {}'
self.logger.error(
err_msg.format(file_manager_type, traceback.format_exc()))
data.outputs.ex_data = err_msg.format(file_manager_type, e)
return False
client = get_client_by_user(executor)
ip_info = cc_get_ips_info_by_str(executor, biz_cc_id, target_ip_list)
ip_list = [{
'ip': _ip['InnerIP'],
'bk_cloud_id': _ip['Source']
} for _ip in ip_info['ip_result']]
file_tags = [_file['tag'] for _file in local_files]
push_result = file_manager.push_files_to_ips(
esb_client=client,
bk_biz_id=biz_cc_id,
file_tags=file_tags,
target_path=target_path,
ips=ip_list,
account=target_account,
callback_url=get_node_callback_url(self.id))
if not push_result['result']:
data.outputs.ex_data = push_result['message']
return False
job_instance_id = push_result['data']['job_id']
data.outputs.job_inst_id = job_instance_id
data.outputs.job_inst_url = get_job_instance_url(
biz_cc_id, job_instance_id)
return True
def _check_and_get_file_manager():
file_manager_type = EnvironmentVariables.objects.get_var(
"BKAPP_FILE_MANAGER_TYPE")
if not file_manager_type:
return False, _("File Manager 未配置，请联系管理员进行配置")
try:
file_manager = ManagerFactory.get_manager(
manager_type=file_manager_type)
except Exception as e:
logger.error(
"[FILE_UPLOAD]can not get file manager for type: {}\n err: {}".
format(file_manager_type, traceback.format_exc()))
return False, str(e)
return True, file_manager
def execute(self, data, parent_data):
executor = parent_data.inputs.executor
biz_cc_id = data.inputs.biz_cc_id
local_files_and_target_path = data.inputs.job_local_files_info[
"job_push_multi_local_files_table"]
target_ip_list = data.inputs.job_target_ip_list
target_account = data.inputs.job_target_account
across_biz = data.get_one_of_inputs("job_across_biz", False)
job_timeout = data.get_one_of_inputs("job_timeout")
task_count = len(local_files_and_target_path)
file_manager_type = EnvironmentVariables.objects.get_var(
"BKAPP_FILE_MANAGER_TYPE")
if not file_manager_type:
data.outputs.ex_data = "File Manager configuration error, contact administrator please."
return False
try:
file_manager = ManagerFactory.get_manager(
manager_type=file_manager_type)
except Exception as e:
err_msg = "can not get file manager for type: {}\n err: {}"
self.logger.error(
err_msg.format(file_manager_type, traceback.format_exc()))
data.outputs.ex_data = err_msg.format(file_manager_type, e)
return False
client = get_client_by_user(executor)
# filter 跨业务 IP
clean_result, ip_list = get_biz_ip_from_frontend(
target_ip_list, executor, biz_cc_id, data, self.logger, across_biz)
if not clean_result:
return False
params_list = [{
"esb_client":
client,
"bk_biz_id":
biz_cc_id,
"file_tags": [
_file["response"]["tag"]
for _file in push_files_info["file_info"]
if _file["response"]["result"] is True
],
"target_path":
push_files_info["target_path"],
"ips":
ip_list,
"account":
target_account,
} for push_files_info in local_files_and_target_path]
if job_timeout:
for param in params_list:
param["timeout"] = int(job_timeout)
# 批量上传请求
if len(params_list) == task_count:
push_results = batch_execute_func(file_manager.push_files_to_ips,
params_list,
interval_enabled=True)
else:
data.outputs.ex_data = _("执行参数为空，请确认")
return False
# 校验请求结果
job_instance_id_list = []
data.outputs.requests_error = ""
for push_object in push_results:
push_result = push_object["result"]
if not push_result["result"]:
err_message = job_handle_api_error(push_result["job_api"],
push_result["kwargs"],
push_result["response"])
self.logger.error(err_message)
data.outputs.requests_error += "{}\n".format(err_message)
else:
job_instance_id_list.append(push_result["data"]["job_id"])
if data.outputs.requests_error:
data.outputs.requests_error = "Request Error:\n{}".format(
data.outputs.requests_error)
data.outputs.job_instance_id_list = job_instance_id_list
# 批量请求使用
data.outputs.job_id_of_batch_execute = job_instance_id_list
data.outputs.job_inst_url = [
get_job_instance_url(biz_cc_id, job_id)
for job_id in job_instance_id_list
]
# 总任务数
data.outputs.task_count = task_count
# 请求成功数
data.outputs.request_success_count = len(job_instance_id_list)
# 执行成功数
data.outputs.success_count = 0
# 所有请求都失败，则返回
if not data.outputs.request_success_count:
data.outputs.ex_data = data.outputs.requests_error
return False
# 任务结果
data.outputs.final_res = task_count == len(job_instance_id_list)
return True
def test_get_nfs_manager(self):
manager = ManagerFactory.get_manager('host_nfs')
self.assertEqual(manager.location, 'BKAPP_NFS_CONTAINER_ROOT')
self.assertEqual(manager.server_location, 'BKAPP_NFS_HOST_ROOT')
def execute(self, data, parent_data):
executor = parent_data.inputs.executor
biz_cc_id = data.inputs.biz_cc_id
local_files = data.inputs.job_local_files
target_ip_list = data.inputs.job_target_ip_list
target_account = data.inputs.job_target_account
target_path = data.inputs.job_target_path
across_biz = data.get_one_of_inputs("job_across_biz", False)
file_manager_type = EnvironmentVariables.objects.get_var(
"BKAPP_FILE_MANAGER_TYPE")
if not file_manager_type:
data.outputs.ex_data = "File Manager configuration error, contact administrator please."
return False
try:
file_manager = ManagerFactory.get_manager(
manager_type=file_manager_type)
except Exception as e:
err_msg = "can not get file manager for type: {}\n err: {}"
self.logger.error(
err_msg.format(file_manager_type, traceback.format_exc()))
data.outputs.ex_data = err_msg.format(file_manager_type, e)
return False
client = get_client_by_user(executor)
# filter 跨业务 IP
clean_result, ip_list = get_biz_ip_from_frontend(
target_ip_list, executor, biz_cc_id, data, self.logger, across_biz)
if not clean_result:
return False
# 这里自动过滤掉上传失败的文件
file_tags = [
_file["response"]["tag"] for _file in local_files
if _file["response"]["result"] is True
]
push_result = file_manager.push_files_to_ips(
esb_client=client,
bk_biz_id=biz_cc_id,
file_tags=file_tags,
target_path=target_path,
ips=ip_list,
account=target_account,
callback_url=get_node_callback_url(self.root_pipeline_id, self.id,
getattr(self, "version", "")),
)
if not push_result["result"]:
err_message = job_handle_api_error(push_result["job_api"],
push_result["kwargs"],
push_result["response"])
self.logger.error(err_message)
data.outputs.ex_data = err_message
return False
job_instance_id = push_result["data"]["job_id"]
data.outputs.job_inst_id = job_instance_id
data.outputs.job_inst_url = get_job_instance_url(
biz_cc_id, job_instance_id)
return True
def test_get_bk_repo_manager(self):
manager = ManagerFactory.get_manager("bk_repo")
self.assertEqual(isinstance(manager.storage, BKRepoStorage), True)