)
from .models import UsersAccount
from permission import perms
from permission.models import PermissionAll
from public.util import libs, rest_framwork_mixin
from message.send_message import sender
from apps.models import AppConnectorRel
from activity.business.models import AccessAlarmStrategy
from monitor.models import MonitorThirdPartyStrategy
jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER
jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER
# Create your views here.
logger = libs.get_logger('users')
class UsersRegisterViewSet(CreateModelMixin, viewsets.GenericViewSet):
"""
create:
用户注册
"""
serializer_class = UsersRegisterSerializer
queryset = UsersAccount.objects.all()
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
self.perform_create(serializer)
data = serializer.data
"""
所有自定义的middleware
"""
import json
import threading
from django.utils.deprecation import MiddlewareMixin
from django.http import HttpRequest, HttpResponse
from .models import OperationGlobalLog
from public.util.libs import get_logger
LOGGER = get_logger('operation.middleware')
class RequestExceptionMiddleware(MiddlewareMixin):
"""
处理错误记录到日志
"""
def process_exception(self, request, exception):
user = str(request.user)
uri = request.path_info
method = request.method
data = request.body.decode()
message = f'''
[Process Exception]
用户: {user}
URI: {uri}
Method: {method}
Data: {data}
Exception: {exception}
'''
LOGGER.error(message)
import json
from .algorithm import DateAlgorithm, NumberAlgorithm, NumberJitterAlgorithm
from monitor.components.falcon_api import (
add_endpoint_to_falcon_grp,
get_grp_hosts,
del_endpoint_to_falcon_grp,
del_endpoint_to_graph,
)
from .vars import (THIRD_PARTY_HG, THIRD_PARTY_PREFIX, MONITOR_DATE_ALGORITHM,
MONITOR_NUMBER_ALGORITHM, MONITOR_NUMBER_JITTER_ALGORITHM)
from public.util.libs import get_logger
from message.send_message import sender
from .vars import NOTICE_DOC_FUNC
logger = get_logger('monitor.components.third_party.libs')
class GetMonitorItemStatus:
"""
获取监控项需要上传的status
:status: 0: 正常，1：异常， 2：失效
"""
def __init__(self, monitor_item, strategy_obj, jitter=False):
self.monitor_item = monitor_item
self.strategy_obj = strategy_obj
self.jitter = jitter
def __get_algorithm(self):
if self.monitor_item in MONITOR_NUMBER_JITTER_ALGORITHM and self.jitter:
return NumberJitterAlgorithm(self.strategy_obj)
"""
cmdb 需要做的定时任务都在这里
"""
import datetime
import json
from cmdb.cloud.models import AliyunKeys, AliyunEcs, AliyunRDS
from cmdb.cloud.aliyun.libs import generate_aliyun_ecs, get_aliyun_resource
from cmdb.cloud.serializers import AliyunEcsUpdateSerializer
from cmdb.native.models import NativeHost
from public.util.libs import get_logger, utc2local, UTC_FORMAT_FULL
from public.util.call_ansible import call_console_ansible_shell
logger = get_logger('cmdb.cmdb_common.cron')
# 通过 aliyun key 更新rds
def aliyun_rds_update_cron():
logger.info('[cron] aliyun_rds_update_cron started')
try:
for keys_obj in AliyunKeys.objects.all():
rds_list = get_aliyun_resource(
access_key=keys_obj.access_key,
access_secret=keys_obj.access_secret,
region_id=keys_obj.region_id,
resource='rds',
kwargs={'region_id': keys_obj.region_id})
logger.debug(f"`aliyun_rds_api()`: {rds_list}")
for rds in rds_list:
try:
try:
rds_obj = AliyunRDS.objects.get(
from rest_framework.response import Response
from rest_framework import permissions
from .serializers import (OpenApiMysqlQuerySerializer,
OpenApiAnsibleHostSerializer,
OpenApiAliyunSLBCtrlSerializer)
from .falcon.libs import analyze_endpoint
from public.util.libs import get_logger, local_mysql_conf
from public.util import call_mysql
from cmdb.cloud.models import AliyunEcs, AliyunKeys
from cmdb.native.models import NativeHost
from cmdb.cloud.aliyun.api import get_aliyun_resource
from common.models import AnsibleHost
from apps.libs import get_app_rel_host
from apps.models import AppDetail
logger = get_logger('openapi.views')
class OpenApiFalconAlarmViewSet(mixins.ListModelMixin,
viewsets.GenericViewSet):
"""
list:
接受 open-falcon 告警回调
"""
def list(self, request, *args, **kwargs):
data = request.query_params
logger.info(f'''falcon 回调信息: {data}''')
try:
if not data:
return Response(data='I am working...')
handle_class = analyze_endpoint(data['endpoint'])
"""
第三方监控，监控算法
"""
import datetime
from abc import ABCMeta, abstractmethod
from .vars import MONITOR_ITEMS, MONITOR_MODALS, THIRD_PARTY_GRAPH_PREFIX
from public.util.libs import get_logger
from monitor.components.falcon_api import get_graph_data
logger = get_logger('monitor.components.third_party.algorithm')
class BaseAlgorithm(metaclass=ABCMeta):
# 指定的监控项
monitor_items = MONITOR_ITEMS
monitor_models = MONITOR_MODALS
def __init__(self, strategy_obj):
self.strategy_obj = strategy_obj
self.result_set = {}
# 异常结果的item id
self.unusual_item_id = []
# 异常结果的item id
self.usual_item_id = []
def op_result(self, op: str, compare_val, standard_val) -> int:
# 1 是触发告警， 0 是正常
logger.debug(f'''{compare_val} {op} {standard_val}''')
result = 1 if eval(f'''{compare_val} {op} {standard_val}''') else 0
print(result)
return result
aliyun_rds_save,
aliyun_nas_save,
aliyun_vpn_save,
aliyun_domain_save,
yuexin_sms_save,
wanweiyiyuan_bank_identity_save,
xuncheng_eryaosu_save,
tencent_message_save,
)
from monitor.models import MonitorThirdPartyStrategy, MonitorThirdPartyJitterStrategy, MonitorTPStrategyItemRel
from monitor.components.falcon_api import push_data_to_open_falcon
from .vars import (THIRD_PARTY_PREFIX, MONITOR_GRAPH_ITEMS, MONITOR_MODALS,
THIRD_PARTY_GRAPH_PREFIX, THIRD_PARTY_GRAPH_HG)
from .libs import GetMonitorItemStatus, push_third_party_notice, push_third_party_jitter_notice
logger = get_logger('monitor.third_party.__init__')
# 保存数据的方法
SAVE_DATA_FUNCS = (
aliyun_ecs_save,
aliyun_domain_save,
aliyun_vpn_save,
aliyun_rds_save,
aliyun_nas_save,
yuexin_sms_save,
wanweiyiyuan_bank_identity_save,
xuncheng_eryaosu_save,
tencent_message_save,
)
# 更新第三方监控数据
aliyun_nas_api,
aliyun_domain_api,
aliyun_vpn_api,
yuexin_sms_api,
wanweiyiyuan_bank_identity_api,
tencent_sms_api,
xuncheng_eryaosu_api,
)
from monitor.models import (MonitorECS, MonitorRDS, MonitorNAS, MonitorDomain,
MonitorVPN, MonitorYueXinSms,
MonitorWanWeiYiYuanBankIdentity, MonitorTencentSms,
MonitorXunChengEryaosu)
from public.util.libs import utc2local, UTC_FORMAT_FULL, get_work_order
from public.util.libs import get_logger
logger = get_logger('monitor.components.third_party.save_data')
def __compare_date(expire_time, now_time=None):
if not now_time:
now_time = datetime.datetime.now()
time_obj = expire_time - now_time
return time_obj.days
def aliyun_ecs_save():
try:
ecs_obj_list = aliyun_ecs_api()
logger.debug(f"`aliyun_ecs_api()`: {ecs_obj_list}")
for ecs_obj in ecs_obj_list:
try:
"""
falcon 回调路由
"""
import json
from abc import ABCMeta, abstractmethod
from message.busniess import ngx_access_doc
from activity.business.models import AccessAlarmResult, AccessAlarmStrategy
from message.send_message import sender
from public.util.libs import get_logger
logger = get_logger('openapi.falcon.libs')
class FalconBaseImplement(metaclass=ABCMeta):
# 处理接收falcon值（用于处理自定义监控值的规范）
# 0: 正常状态
# 1: 异常状态
# 2: 数据失效
def falcon_receive_value(self, value):
falcon_status = {0: 'ok', 1: 'problem', 2: 'expire'}
return falcon_status.get(int(value), 'none')
def send(self, detail_obj, **kwargs):
if 1 == detail_obj.is_mail:
kwargs['send_type'] = 'mail'
sender(**kwargs)
if 1 == detail_obj.is_message:
kwargs['send_type'] = 'message'
sender(**kwargs)
# coding: utf-8
"""
access log 告警， 主要通过判断 request cost time
"""
import datetime
import requests
import numpy
import json
import asyncio
from django.conf import settings
from activity.business.models import AccessAlarmStrategy, AccessAlarmResult
from activity.business.vars import NGX_ACCESS_FALCON_PREFIX
from public.util.libs import local2utc, UTC_FORMAT_FULL, datetime2timestamp, get_logger
from monitor.components.falcon_api import push_data_to_open_falcon
logger = get_logger('activity.business.ngx_access_alarm.main')
# Falcon endpoint prefix
EP_PREFIX = NGX_ACCESS_FALCON_PREFIX
# ES access log index
ES_INDEX = 'plog-callchain*'
# 获取的使用sql插件es地址
SQL_ES_ADDRESS = settings.DEPLOY_CONF.get('business', 'sql_es_address')
# request header
HEADERS = {'Content-Type': 'application/json'}
# 返回UTC 时间
def get_current_fmt_time(now_time_obj):
now = local2utc(now_time_obj)
return now.strftime(UTC_FORMAT_FULL)
from rest_framework import permissions
from permission import perms
from .models import AliyunKeys, AliyunEcs, AliyunRDS
from ..cmdb_common.models import TagsAliyunEcsRel
from .serializers import (AliyunKeysSerializer, AliyunPureKeysSerializer,
AliyunEcsAutoSerializer, AliyunEcsUpdateSerializer,
AliyunEcsSerializer, AliyunKeys2Serializer,
AliyunRdsSerializer)
from .aliyun.libs import generate_aliyun_ecs
from .aliyun.api import get_aliyun_resource
from .filters import AliyunGraphFilter, AliyunKey2ECsFilter, AliyunRdsGraphFilter, AliyunKey2RdsFilter
from apps.models import AppHostRel, AppDetail
from public.util.libs import get_logger, local2utc, UTC_FORMAT_NO_SEC, utc2local, E_CHARTS_FORMAT, UTC_FORMAT_FULL
from public.util import rest_framwork_mixin
logger = get_logger('aliyun.views')
# -------------------- 阿里云 --------------------
class AliyunKeyViewSet(mixins.ListModelMixin,
rest_framwork_mixin.PureUpdateModelMixin,
mixins.DestroyModelMixin, mixins.CreateModelMixin,
viewsets.GenericViewSet):
"""
list:
获取 aliyun access keys 信息
update:
更新 aliyun access key的部分或所有信息
delete:
删除 aliyun access key 信息
create:
"""
app alive 信息
"""
import datetime
import requests
import json
from urllib.parse import urlparse
from django.conf import settings
from monitor.components.falcon_api import get_open_falcon_login_token
from public.util import call_mysql
from public.util.libs import local_mysql_conf, get_logger
logger = get_logger('monitor.components.app_alive')
class AppAliveGraph:
"""
获取 app alive 监控信息流程:
env -> urlooker(获取strategy id) --> open-falcon(urlooker_id)
"""
def __init__(self,
environment=None,
app_id=None,
check_api=None,
hours=12):
self.env = environment
self.app_id = app_id
self.check_api = check_api
self.hours = hours
# 这里只能从配置文件获取，无法从urlooker 直接获取
self.__parse_agent = {
"""
Crontab
"""
import requests
import json
import datetime
from .models import AppALiveBrief, AppAliveStatistics
from public.util.libs import get_logger
from public.util import call_mysql
from apps.models import AppDetail
from monitor.components.app_alive import AppAliveGraph
logger = get_logger('common.cron')
# 收集app每天的错误信息点, 只针对release 环境，用于展示在dashboard
def collect_app_alive_failed_point_cron():
with call_mysql.OpsMysqlClient() as cursor:
sql = '''select `url`, `ops_cp_app_id` from urlooker.strategy where environment = 'release';'''
cursor.execute(sql)
result_set = cursor.fetchall()
for app_obj in result_set:
try:
app_d_obj = AppDetail.objects.get(id=app_obj['ops_cp_app_id'])
app_alive_graph_obj = AppAliveGraph(
environment='release',
app_id=app_obj['ops_cp_app_id'],
check_api=app_obj['url'],
hours=24)
monitor_info_list = app_alive_graph_obj.monitor_info()
failed_point_count = 0
from .models import Tags, TagsAliyunEcsRel, TagsNativeHostRel
from .serializers import (
TagsSerializer,
TagsEcsRelSerializer,
TagsNativeHostRelSerializer,
AnisbleUpdateHostInfoSerializer,
AnsibleAddHostInfoSerializer,
)
from ..cloud.models import AliyunEcs
from ..native.models import NativeHost
from public.util.libs import get_logger
from rest_framework.response import Response
from .libs import ansible_update_aliyun_ecs, ansible_update_native_host
from .filters import TagsNativeHostRelFB
logger = get_logger('cmdb_common.views')
# Create your views here.
class TagsViewSet(mixins.ListModelMixin, mixins.CreateModelMixin,
mixins.DestroyModelMixin, viewsets.GenericViewSet):
"""
list:
获取所有标签
create:
创建标签
destroy:
删除标签及其关系数据
"""
queryset = Tags.objects.all()
from public.util.libs import get_logger, local_mysql_conf
from .models import AppDetail, AppHostRel, AppConnectorRel
from cmdb.cloud.models import AliyunEcs
from cmdb.native.models import NativeHost
from .serializers import AppDetailSerializer, AppHostRelSerializer, AppAliveUrlookerSerializer
from .libs import get_app_rel_host
from .sql import (
create_urlooker_app_alive,
update_urlooker_app_alive,
delete_urlooker_app_alive,
select_urlooker_app_alive,
select_urlooker)
from monitor.components.falcon_api import del_endpoint_to_graph
logger = get_logger('apps.view')
# Create your views here.
class AppDetailViewSet(mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.CreateModelMixin,
rest_framwork_mixin.PureUpdateModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet):
"""
list:
获取所有 app 详情列表
retrieve:
获取单个 app 详情
timestamp2format_time,
LOCAL_FORMAT)
from public.util import call_mysql
from public.util.rest_framwork_mixin import PureUpdateModelMixin
from .components.third_party.vars import (
MONITOR_MODALS,
MONITOR_GRAPH_ITEMS,
THIRD_PARTY_GRAPH_PREFIX,
MONITOR_ITEMS)
from monitor.components.falcon_api import get_graph_data
from .components.third_party import save_third_party_data
from monitor.components.falcon_api import del_endpoint_to_graph
logger = get_logger('monitor.views')
# Create your views here.
class MonitorAppAliveGraphViewSet(mixins.RetrieveModelMixin,
viewsets.GenericViewSet):
"""
retrieve:
获取单个APP alive 12 个小时的监控信息
"""
filter_class = MonitorAppAliveFilter
queryset = AppDetail.objects.all()
permission_classes = [permissions.IsAuthenticated, perms.IsPagePermissionRW]
from .ngx_access_alarm import ngx_access_launch
from public.util.libs import get_logger
logger = get_logger('activity.business.cron')
def ngx_access_cron():
logger.info('[cron] ngx_access_cron started.')
try:
ngx_access_launch()
except Exception as e:
logger.exception(e)