def fast_push_file(self, kwargs):
"""
快速分发文件
"""
kwargs.update({
"bk_username": self.bk_username,
})
result = JobV3Api.fast_transfer_file(kwargs, raw=True)
if result["result"]:
query_kwargs = {
"job_instance_id": result["data"].get("job_instance_id"),
"bk_biz_id": kwargs.get("bk_biz_id"),
}
result = self.get_task_result_status(query_kwargs)
logger.info(
build_job_exec_log_format(self.bk_username, 'fast_push_file',
kwargs['task_name']))
return result
else:
logger.error(
build_job_err_log_format(self.bk_username, 'fast_push_file',
kwargs, result))
return None
def create_record_detail(self, task_type):
"""
create_record_detail方法：存储任务记录
@param task_type: 内部定义的任务流程id, 每个id都会对应不同的任务名称，这样方便后端统一管理，并节省数据库存储字段的长度，优化搜索
"""
task_info = copy.deepcopy(self.param_info)
# 屏蔽 password 等字段，防止录入到数据库
if task_info.get('password'):
task_info.pop('password')
logger.info(self.param_info)
self.task_id = create_record_detail(
{
"db_type": self.db_type,
"app_id": self.param_info['app_id'],
"cluster_name": self.param_info['cluster_name'],
"task_type": task_type,
"task_mode": 1,
"pipeline_id": "000",
"op_user": self.param_info['bk_username'],
"task_params": task_info,
}
)
if self.task_id == 0:
logger.error("task表生成失败，任务结束")
return False
return True
def fast_execute_script(self, kwargs):
"""
快速执行脚本
"""
# shell 脚本内容需要base64编码
kwargs.update({
"bk_username": self.bk_username,
})
result = JobV3Api.fast_execute_script(
{
**kwargs,
**fast_execute_script_common_kwargs
}, raw=True)
if result["result"]:
query_kwargs = {
"job_instance_id": result["data"].get("job_instance_id"),
"bk_biz_id": kwargs.get("bk_biz_id"),
}
result = self.get_task_result_status(query_kwargs)
logger.info(
build_job_exec_log_format(self.bk_username,
'fast_execute_script',
kwargs['task_name']))
return result
else:
logger.error(
build_job_err_log_format(self.bk_username,
'fast_execute_script', kwargs,
result))
return None
def home(request):
"""
@summary: 移动端首页
"""
# 蓝鲸用户
bk_username = request.user.username
# 微信用户
wx_username = model_to_dict(request.weixin_user)
logger.info('bk_username:{bk_username}, wx_username:{wx_username}'.format(
bk_username=bk_username, wx_username=wx_username))
return render(request, 'weixin/index.html')
def consume_omnibus_overtime():
try:
topics = KAFKA.get('topics')
broker_list = KAFKA.get('broker_list')
consumer = KafkaConsumer(
bootstrap_servers=broker_list,
group_id='celery_topic_poll',
consumer_timeout_ms=20000,
)
for topic in topics:
partitions = consumer.partitions_for_topic(topic)
logger.info('主题%s对应分区为%s' % (topic, partitions))
for partition in partitions:
tp = TopicPartition(topic, partition)
consumer.assign([tp])
logger.info(consumer.end_offsets([tp]))
# next_offset = consumer.end_offsets([tp_fault]).get(tp_fault)
# consumer.seek(tp_fault, int(next_offset) - 1)
msg = consumer.poll(timeout_ms=50)
if not msg:
logger.info(str(msg))
else:
records = msg.get(tp)
for record in records:
into_database(record)
consumer.commit()
except Exception as err:
logger.info(str(err))
def consume_fault_location():
try:
topic = 'HXB_Fault_Location'
broker_list = KAFKA.get('broker_list')
consumer = KafkaConsumer(
bootstrap_servers=broker_list,
group_id='celery_fault_location_poll',
consumer_timeout_ms=20000,
)
partitions = consumer.partitions_for_topic(topic)
logger.info('主题%s对应分区为%s' % (topic, partitions))
for partition in partitions:
tp_fault = TopicPartition(topic, partition)
consumer.assign([tp_fault])
logger.info(consumer.end_offsets([tp_fault]))
# next_offset = consumer.end_offsets([tp_fault]).get(tp_fault)
# consumer.seek(tp_fault, int(next_offset) - 1)
msg = consumer.poll(timeout_ms=50)
if not msg:
logger.info(str(msg))
else:
records = msg.get(tp_fault)
for record in records:
logger.info(
f'当前记录{record.topic, record.partition, record.offset}')
if not os.getenv('BK_ENV'):
data = eval(
zlib.decompress(base64.b64decode(
record.value)).decode('gbk'))
else:
data = json.loads(record.value.decode('utf-8'))
fault_location = FaultLocation.objects.filter(
topic=record.topic,
partition=int(record.partition),
offset=int(record.offset))
if fault_location:
fault_location[0].offset = int(record.offset)
fault_location[0].value = str(data)
fault_location[0].save()
else:
new_fault = FaultLocation(topic=record.topic,
partition=int(
record.partition),
offset=int(record.offset),
value=str(data))
new_fault.save()
consumer.commit()
except Exception as err:
logger.info(str(err))
def consume_omnibus_overtime():
try:
topics = KAFKA.get('topics')
broker_list = KAFKA.get('broker_list')
consumer = KafkaConsumer(
bootstrap_servers=broker_list,
group_id='api_data_subscribe_omnibus_overtime',
# consumer_timeout_ms=20000,
)
for topic in topics:
partitions = consumer.partitions_for_topic(topic)
logger.info('主题%s对应分区为%s' % (topic, partitions))
consumer.subscribe(topics=topics)
for record in consumer:
# logger.info(datetime.now(), record.topic, record.partition, record.offset)
into_database(record)
except Exception as err:
logger.info(str(err))
def execution(self, task, task_id, user):
self.ip_error_list.clear()
exe = ExecuteLog.objects.create(work_code=str(uuid.uuid1()),
exec_state=1,
task_id=self.task_id,
operator=self.user)
for index, quota in enumerate(self.refrefsh_quota_list()):
# 1
task_result_obj = TaskResult.objects.create(
task=task,
task_state=0,
task_step_state=0,
quota_id=int(quota['id']),
)
task_result = TaskResult.objects.filter(id=task_result_obj.id)
# 执行前修改表状态
# 任务结果表
task_result.update(task_state=1)
self.execute_script(task_result_obj, task_result, quota, exe.id)
# 判断是否是脚本列表的最后一组数据
if index >= len(self.refrefsh_quota_list()) - 1:
dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 任务结果表
task_result.update(task_state=2)
# 任务表
Task.objects.filter(id=task.id).update(exec_state=0)
# 删除redis缓存状态
redis_link.hdel('current_task', self.task_id)
# 历史记录表
ExecuteLog.objects.filter(id=exe.id).update(exec_state=2,
end_time=dt)
redis_link.delete("current_task")
# 通知
notify_result = send_notify(task_id)
logger.info(
f"发送通知 总状态({notify_result['result']}) 发送方式({notify_result['send_type']}) 发送详情({notify_result['send_info']})"
)
return
def into_database(record):
# logger.info(record)
data = zlib.decompress(base64.b64decode(record.value)).decode('gbk')
if not os.getenv('BK_ENV'): # 开发环境判断
data = eval(data)
else:
data = json.loads(data)
# logger.info(len(data))
for item in data:
if not item.get('children'):
hash_id = hashlib.md5(
str(item).encode(encoding='UTF-8')).hexdigest()
params = {'hash_id': hash_id}
if item.get('summary'):
# logger.info('***omnibus***', str(item))
model_obj = lens._registry.get(EmergencyOmnibus)
response = model_obj._api(method='GET', data=params)
if response['code'] == 1:
response_data = response.get('data')
if not response_data.get('items'):
for key, value in item.items():
if key in EmergencyOmnibus._meta.field_map.keys():
params[EmergencyOmnibus._meta.field_map.get(
key)] = value
else:
params[key] = value
params['operation'] = int([
choice[0] for choice in choices.get(
'EmergencyOmnibus_opration')
if choice[1] == params['operation']
][0])
params['level'] = '0' if params['level'] not in {
'10', '20', '40', '50'
} else params['level']
# logger.info('omnibus params', params)
response = model_obj._api(method='POST', data=params)
if response['code'] != 1:
logger.info(
f'综合类记录{record.topic, record.partition, record.offset}写入失败',
str(response))
elif response_data.get('items')[0].get(
'hash_id') == hash_id:
logger.info(
f'综合类记录{hash_id}已存在{record.topic, record.partition, record.offset}'
)
else:
logger.info(str(response))
if item.get('key'):
model_obj = lens._registry.get(EmergencyOvertime)
response = model_obj._api(method='GET', data=params)
if response['code'] == 1:
response_data = response.get('data')
if not response_data.get('items'):
for key, value in item.items():
if key in EmergencyOvertime._meta.field_map.keys():
params[EmergencyOvertime._meta.field_map.get(
key)] = value
else:
params[key] = value
code_code_cn = params.pop('code')
rtcode_rtcode_cn = params.pop('rtcode_cn')
params['code'] = code_code_cn.split(':')[0]
params['code_cn'] = code_code_cn.split(':')[1]
params['rtcode'] = rtcode_rtcode_cn.split(':')[0]
params['rtcode_cn'] = rtcode_rtcode_cn.split(':')[1]
# logger.info('overtime params', params)
response = model_obj._api(method='POST', data=params)
if response['code'] != 1:
logger.info(
f'交易类记录{record.topic, record.partition, record.offset}写入失败',
str(response))
elif response_data.get('items')[0].get(
'hash_id') == hash_id:
logger.info(
f'交易类记录{hash_id}已存在{record.topic, record.partition, record.offset}'
)
else:
logger.info(str(response))
def myFunc(request):
logger.info("这是提示信息")
logger.debug("这是debug信息")
logger.warning("这是警告信息")
logger.error('这是错误信息')
return render(request, 'mytestapp/index.html', {})
def info(msg, *args, **kwargs):
bk_logger.info(msg, *args, **kwargs)
if ISDEV:
print(msg, *args)