import json
import datetime
import subprocess
from app_fuzhou.models import TrustLog, BlackboxHost, Virusbook, WhiteList, OperationHost
from app_fuzhou.views_utils.logger import logger
from app_fuzhou.views_utils.localconfig import JsonConfiguration
from app_fuzhou.views_utils.clamav.clamd_scan import ClamdScanner
from app_fuzhou.views_utils.rpc.blackbox.black_box_client import BlackBoxRPCClient
from app_fuzhou.views_utils.global_config import GlobalConf
from django.db import transaction
from django.db.models import Q
from django.forms.models import model_to_dict
GLOBAL_CONFIG = GlobalConf()
LOCAL_CONFIG = JsonConfiguration()
BLACKBOX_RPC_IP = LOCAL_CONFIG.blackbox_ip
BLACKB0X_RPC_PORT = LOCAL_CONFIG.used_ports['blackbox_rpc']
class BaseMessage(object):
def __init__(self):
self.Head = "8LABjs0nHe@dBegin"
self.Action = ""
self.Ip = ""
self.Host = ""
self.Content = ""
self.Date = datetime.datetime.now().strftime("%Y-%m-%d %X")
self.End = "8LABjs0nHe@dEnd"
def get_waf_log_aggregations_week():
"""
获取包括今天在内的7天里的每天的各个级别的日志的数量,5个级别*7天
获取各个级别日志总数
:return: week:{date:[],dos-attack:[],...},total: {dos-attack:0,....}
"""
days = []
today = datetime.date.today()
for _day in range(6, -1, -1): # 日子排序为从远到近
days.append(
(today -
datetime.timedelta(days=_day)).strftime("%Y-%m-%d")) # 近7天
global_config = GlobalConf()
hosts = LOCAL_CONFIG.client_audit_hosts
es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port)
index_list = []
"""
for host in hosts: # 生成索引列表
index_list.append(index + host['ip'])
"""
level_list = [
"web-attack", "sensitive-data-tracking", "identification-error",
"dos-attack", "http-defense"
]
week_aggr_dict = {'date': days}
total_count = {}
# 首先解析flag,从配置文件查找flag对应字段
for _level in level_list:
body = {
"query": {
"bool": {
"must": [
{
"match_phrase": {
"_type": "wafLog"
}
}, # 必须匹配规则
],
"should": [],
"minimum_should_match": 1, # 后面加的message匹配规需要至少匹配一个
}
},
"size": 0,
"aggs": { # 聚合
"week_history": {
"date_histogram": {
"field": "@timestamp", # 时间字段
"interval": "day", # 按天统计
"format": "yyyy-MM-dd",
"min_doc_count": 0,
"time_zone": "+08:00", # es默认时区是UTC,所以需要+8
"extended_bounds": {
"min": days[0],
"max": days[6]
} # 范围为近7天内,包括今天
}
}
}
}
week_aggr_dict[_level] = []
if _level == "web-attack": # web-attack
rules = global_config.RULES['EXPERIMENTAL_RULES']
elif _level == "sensitive-data-tracking": # sensitive-data-tracking
rules = global_config.RULES['OPTIONAL_RULES']
elif _level == "identification-error": # identification-error
rules = global_config.RULES['SLR_RULES']
elif _level == "dos-attack": # dos-attack
rules = global_config.RULES['DOS_RULES']
else: # http-defense
rules = global_config.RULES['BASE_RULES'] # 匹配http规则
except_rules = [] # 或者不匹配其他任何
except_rules += global_config.RULES['EXPERIMENTAL_RULES']
except_rules += global_config.RULES['OPTIONAL_RULES']
except_rules += global_config.RULES['SLR_RULES']
except_rules += global_config.RULES['DOS_RULES']
must_not_list = []
body["query"]["bool"]["should"].append(
{"bool": {
"must_not": must_not_list
}})
for rule in except_rules: # 把message匹配规则加入到body中
must_not_list.append({"match_phrase": {"message": rule}})
for rule in rules: # 把message匹配规则加入到body中
body["query"]["bool"]["should"].append(
{"match_phrase": {
"message": rule
}})
try:
results = es.search(index=WAF_INDEX,
body=body,
ignore_unavailable=True) # 从es中读取
total_count[_level] = results['hits']['total'] # 记录每种类型的总数
# 虽然,上面的聚合操作指定了只取最近７天,但其只能进行扩展,不能进行压缩,因此,下面的切片操作[-7:]是必须的!!
for _result in results['aggregations']['week_history']['buckets'][
-7:]: # ES返回的聚合数据list有序,顺序为从小到大
week_aggr_dict[_level].append(
_result['doc_count']) # 存入顺序为 按照日期 从小到大
# total_count[_level] = results['hits']['total'] # 记录每种类型的总数
except Exception as e:
logger.error(e)
if not week_aggr_dict[
_level]: # 当索引不存在时,week_aggr_dict[_level]就是一个空列表,此时进行如下赋值
week_aggr_dict[_level] = [0, 0, 0, 0, 0, 0, 0]
return week_aggr_dict, total_count
def get_waf_log_from_es(attack_type,
page=1,
pagesize=50,
time_filter=None,
sort="desc"):
"""
从ES获取WafLog
:param attack_type: waf攻击类型
:param page: 第几页
:param pagesize: 每页多少条
:param time_filter: 时间过滤器
:param sort: 排序
:return:
"""
global_config = GlobalConf()
es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port)
body = {
"query": {
"bool": {
"must": [
{
"match_phrase": {
"_type": "wafLog"
}
}, # 必须匹配规则
],
"should": [],
"minimum_should_match": 1, # 后面加的message匹配规需要至少匹配一个
# "filter": {"range": {"@timestamp": {"gte": "now-3d", "lte": "now"}}} # 时间过滤器
}
},
"from": (page - 1) * pagesize,
"size": pagesize,
}
if sort == "desc" or sort == "asc":
body["sort"] = {"@timestamp": sort}
if time_filter:
body["query"]["bool"]['filter'] = time_filter
# 首先解析flag,从配置文件查找flag对应字段
if attack_type == "web-attack": # web-attack
rules = global_config.RULES['EXPERIMENTAL_RULES']
elif attack_type == "sensitive-data-tracking": # sensitive-data-tracking
rules = global_config.RULES['OPTIONAL_RULES']
elif attack_type == "identification-error": # identification-error
rules = global_config.RULES['SLR_RULES']
elif attack_type == "dos-attack": # dos-attack
rules = global_config.RULES['DOS_RULES']
elif attack_type == "http-defense": # http-defense
rules = global_config.RULES['BASE_RULES'] # 包含http规则
except_rules = [] # 或者不包含其他的
except_rules += global_config.RULES['EXPERIMENTAL_RULES']
except_rules += global_config.RULES['OPTIONAL_RULES']
except_rules += global_config.RULES['SLR_RULES']
except_rules += global_config.RULES['DOS_RULES']
must_not_list = []
for rule in except_rules: # 把must_not匹配规则加入到body中
must_not_list.append({"match_phrase": {"message": rule}})
body["query"]["bool"]["should"].append(
{"bool": {
"must_not": must_not_list
}})
else:
return [], 0
for rule in rules: # 把message匹配规则加入到body中
body["query"]["bool"]["should"].append(
{"match_phrase": {
"message": rule
}})
"""
for host in hosts: # 生成索引列表
index_list.append(index+host['ip'])
"""
try:
result = es.search(index=WAF_INDEX, body=body,
ignore_unavailable=True) # 从es中读取
except Exception as e:
logger.error(e)
return [], 0
# 至此从es上获得了数据
return result['hits']['hits'], result['hits']['total']
import django
from django.db import connection
from django.test import Client
from app_fuzhou.util import mysql_base_api
from app_fuzhou.views_utils.global_config import GlobalConf
os.environ['DJANGO_SETTINGS_MODULE'] = 'octastack_fuzhou_web.settings'
django.setup() # 添加的代码
client = Client()
response = client.post('api/get_scan_config',)
print(response.content)
gc = GlobalConf()
# Function: test interface with Django connection
def django_test():
mysql_base_api.django_setup('database_base_api', 'base_api.settings')
f = open("6.json")
scripts = json.load(f)
mysql_base_api.base_api(connection, connection.cursor(), scripts)
f.close()
connection.close()
def uniform_test(conn, cursor, filename):
f = open(filename)
json_data = json.load(f)