def set_scanport_config(host_ip, config):
"""
设置端口扫描的配置
:return: 是否设置成功
"""
# 92-95 88888
# sql = "INSERT INTO system_config(host_ip, type, config) VALUES(%s, " \
# "%s, %s) ON DUPLICATE KEY UPDATE config=%s"
# conn = None
# cursor = None
try:
# 99-103 88888
# conn, cursor = mysql_base_api.sql_init(
# CONFIG.mysql_host, CONFIG.mysql_user,
# CONFIG.mysql_pass, CONFIG.mysql_database
# )
# mysql_base_api.sql_execute(conn, cursor, sql, [host_ip, TYPE, config, config])
SystemConfig.objects.update_or_create(host_ip=host_ip, type=TYPE, defaults={'config': config})
# 相当于先去 get(host_ip=host_ip, type=TYPE) 如果存在更新 config
# 不存在 create(host_ip=host_ip, type=TYPE, config=config)
return True
except Exception as e:
logger.error(e)
return False
def update_op_host(op_type, mac, exe_path):
"""
更新可信运维主机
:param op_type:
:param mac:
:param exe_path:
:return:
"""
result = {"status": 'error'}
try:
if op_type == "create":
OperationHost.objects.create(id=None,
mac_address=mac,
exe_path=exe_path)
result['status'] = 'success'
elif op_type == "delete":
objs = list(OperationHost.objects.filter(mac_address=mac))
if len(objs) > 0:
objs[0].delete()
result['status'] = 'success'
except Exception as e:
result['status'] = 'error'
logger.error(e)
return result
def get_scan_config(host_ip):
"""
获取制定主机的扫描配置
:param host_ip: 主机IP,
:return: 配置信息
"""
conn = None
cursor = None
default = "{}"
try:
conn, cursor = mysql_base_api.sql_init(CONFIG.mysql_host,
CONFIG.mysql_user,
CONFIG.mysql_pass,
CONFIG.mysql_database)
sql = "SELECT config FROM system_config WHERE host_ip=%s and type=%s"
result = mysql_base_api.select_one_row(cursor, sql, [host_ip, TYPE])
if result is None:
return default
return result.get('config', default)
except Exception as e:
logger.error(e)
finally:
if conn is not None and cursor is not None:
mysql_base_api.sql_close(conn, cursor)
def get_watcherlab_daily(n):
"""
获取每天总的威胁情报数量
:return: 返回近n天攻击信息
"""
result = OrderedDict()
try:
es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port)
# 对每天情报数量进行聚合
body = {
"size": 0,
"aggs": {
"date": {
"date_histogram": {
"field": "@timestamp",
"interval": "day",
"format": "yyyy-MM-dd"
}
}
}
}
_result = es.search(index='watcherlab-*',
body=body,
ignore_unavailable=True) # 从es中读取
_result = _result['aggregations']['date']["buckets"]
items = _result[-n:] if len(_result) > n else _result
for item in items:
result[item["key_as_string"]] = item["doc_count"]
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
finally:
return result
def virus_init():
# 威胁分析结果存入virusbook表中
update_virus_book()
path_value = []
state_code = 2 # 判断是否有威胁的标志
white_list = []
for item in Virusbook.objects.all():
if item.fileName:
path_value.append(item.fileName)
if item.isWhite == 0:
state_code = 1
else:
# 将微步认为是白名单的信息添加在表whitelist中
white_list_item = WhiteList(client_name=item.host,
ip=item.ip,
file_router=item.fileName,
file_data=item.resource)
white_list.append(white_list_item)
# 将微步认为是白名单信息的记录通过rpc发送给blackbox
save_whitelog(model_to_dict(white_list_item))
# 并在trustlog中设置level为DEBUG
TrustLog.objects.filter(
host=item.host, ip=item.ip,
filename=item.fileName).update(level='DEBUG')
# 在virusbook中删除该条记录
Virusbook.objects.filter(id=item.id).delete()
WhiteList.objects.bulk_create(white_list)
path_value.append(state_code)
(status, output) = subprocess.getstatusoutput("/8lab/blackbox -w")
if status != 0:
logger.error("[BLACKBOX][ERROR] blackbox reset white list failed!")
return path_value
def handle_get_waf_log(results, attack_type):
"""
处理wafLog,记录waf攻击的攻击源,攻击目标,攻击时间,攻击工具,攻击类型信息
:param results:
:param attack_type: 攻击类型
:return:
"""
logs = []
time_delta = datetime.timedelta(hours=8)
for hit in results:
try:
log = {
"inter_ip":
hit['_source']['type'], # 攻击目标ip
"inter_time": (datetime.datetime.strptime(
hit["_source"]["@timestamp"], "%Y-%m-%dT%H:%M:%S.%fZ") +
time_delta).strftime('%Y-%m-%d %H:%M:%S'),
"inter_source":
hit['_source'].get('XForwardedFor', ''), # 攻击源ip
"inter_tool":
hit['_source'].get('Referer', ''), # 攻击来源
"defend_type":
attack_type
} # 防御类型
logs.append(log)
except Exception as e:
logger.error(str(e))
logs.sort(key=lambda x: x["inter_time"],
reverse=True) # 之前已经经过反序,这一步仍检测一次是否有乱序
return logs
def get_details_time(time=5, limit=100):
"""
获取一定时间内的攻击信息
:param time: 秒, 时间长度
:param limit: 分页大小
:return: 该时间段内的攻击信息
"""
conn = None
cursor = None
try:
conn = get_connection()
cursor = conn.cursor(MySQLdb.cursors.DictCursor)
sql = "SELECT source_ip, des_ip, attack_type, occur_time " \
"FROM attack_ip WHERE occur_time " \
"BETWEEN date_sub(now(), INTERVAL %d MINUTE) AND now() LIMIT %d" \
% (time, limit)
# logger.info(sql)
cursor.execute(sql)
query_result = cursor.fetchall()
return query_result
except Exception as e:
logger.error(e)
finally:
cursor.close()
conn.close()
def black_list_status(request):
"""
返回黑名单（恶意行为）
:param request:
参数1: shenji_ip
参数2: type 3为黑名单行为
:return:
"""
# 接收参数
audit_ip = request.POST.get("shenji_ip", "#")
_type = request.POST.get("type")
return_dict = {} # 返回的字典
hosts = jc.db_ip_list
ips = []
for host in hosts:
ips.append(host['ip'])
# audit_ip 获取不到就用 ips 里的第一个ip
audit_ip = ips[0] if audit_ip == "#" else audit_ip
try:
# 把黑名单行为（恶意行为）放入到 black_action中
black_action = list(WhiteUserAction.objects.filter(
ip=audit_ip, type=_type).order_by('-id').values('sid', 'value'))
# id降序查询 结果 [{'sid': sid1, 'value': value1}, {'sid': sid2, 'value': value2}]
return_dict['action'] = black_action
except Exception as e:
logger.error(e)
return HttpResponse(json.dumps(return_dict))
def send_message(content):
"""
发送短信
:return:
"""
req = top.api.AlibabaAliqinFcSmsNumSendRequest()
req.set_app_info(
top.appinfo("23827310", "24dbb9f199ea5fbc2826e4f2662c15df"))
req.extend = ""
req.sms_type = "normal"
req.sms_free_sign_name = "八分量持续免疫系统"
if content == "":
req.sms_param = ""
req.sms_template_code = "SMS_67765123"
else:
req.sms_param = {'errorinfo': content}
req.sms_template_code = "SMS_67715193"
req.rec_num = "13488690980,17610000528"
try:
req.getResponse()
except Exception as e:
if hasattr(e, "submsg"):
logger.error("短信发送失败：" + e.submsg)
else:
logger.error(e)
def white_list_status(request):
"""
返回白名单状态
:param request: shenji_ip
:return:
"""
return_dict = {} # 返回的字典
hosts = jc.db_ip_list
ips = []
for host in hosts:
ips.append(host['ip'])
audit_ip = request.POST.get("shenji_ip", "#") # 获取ip
# db_type = request.POST.get("db_type").lower() # 获取数据库类型
audit_ip = ips[0] if audit_ip == "#" else audit_ip
# 获取数据库的添加的用户白名单 和 数据库行为白名单
user_action_query = WhiteUserAction.objects.filter(ip=audit_ip).order_by('-id')
# 从white user表中查询所有的user_name，从white action表中查询所有的 action_name
# 把用户白名单放入到white_user中，把用户行为白名单放入到white_action中
# 当type=1时 返回数据 ['value1', 'value2', 'value3']
# 当type=2时 返回数据 [{'sid': sid1, 'value': value1}, {'sid': sid2, 'value': value2}]
try:
# flat 为True，它表示返回的结果为单个值而不是元组 ['value1', 'value2', 'value3']
return_dict['user'] = list(user_action_query.objects.filter(
type=1).values_list('value', flat=True))
return_dict['action'] = list(user_action_query.objects.filter(
type=2).values('sid', 'value'))
except Exception as e:
logger.error(e)
return HttpResponse(json.dumps(return_dict))
def delete_rules(request):
"""
黑名单行为删除（恶意行为）
:param request:
参数1: values
参数2: ip 审计页面－当前ip
:return:
"""
response_result = {'code': '201'}
# 接收参数
values = request.POST.get("values", "")[:-1] # 前台数据 ,由于最后多了一个"#号",所以需要删除
ip = request.POST["ip"]
sids = values.split("#")
# 数据库删除黑名单行为
try:
snort_rpc = SnortRpcClient(ip, 30303)
rpc_result = snort_rpc.del_rules(sids)
if rpc_result:
WhiteUserAction.objects.filter(sid__in=sids, ip=ip).delete()
response_result = {'code': '200'}
except Exception as e:
response_result = {'code': '201'}
logger.error(e)
return HttpResponse(json.dumps(response_result))
return HttpResponse(json.dumps(response_result))
def clam_scan(self, invoke_ip, conf_str):
"""
执行病毒扫描的线程
:param invoke_ip:
:return:
"""
global SCAN_KILLED
try:
# 更新数据扫描标记位
# MachineList.objects.filter(hostip=invoke_ip).update(is_scan=True)
# 开始扫描
result = clamav.clam_scan(invoke_ip, conf_str)
if len(result) is not 0 and not SCAN_KILLED:
# 将扫描日志和标记位更新
conn, cursor = mysql_base_api.sql_init(LOCAL_CONFIG.mysql_host,
LOCAL_CONFIG.mysql_user,
LOCAL_CONFIG.mysql_pass,
LOCAL_CONFIG.mysql_database,
LOCAL_CONFIG.mysql_port) # 初始化数据库
# 从数据库中查询
sql = 'update app_fuzhou_machinelist set is_scan=0,scan_log="%s" where hostip="%s"' \
% (result, invoke_ip)
mysql_base_api.sql_execute(conn, cursor, sql, "")
# 关闭数据库
mysql_base_api.sql_close(conn, cursor)
# MachineList.objects.filter(hostip=invoke_ip).update(
# scan_log=str(result), is_scan=False)
except Exception as e:
logger.error(e)
finally:
try:
MachineList.objects.filter(hostip=invoke_ip).update(is_scan=False)
except Exception as e:
logger.error(e)
def __new__(cls, *args, **kwargs):
lock = Lock()
logger.debug("init rabbitMQ instance")
if cls._instance is None:
try:
lock.acquire() # 上锁, 用于保证线程安全性
# 双重锁
if cls._instance is None:
cls._instance = super(RabbitMQConnection, cls).__new__(cls)
credentials = pika.PlainCredentials('8lab', '8lab')
cls._connection = pika.BlockingConnection(
# heartbeat_interval=0表示关闭server的心跳检测, 防止超时关闭
pika.ConnectionParameters(
host=LOCAL_CONFIG.rabbitmq_server,
port=LOCAL_CONFIG.rabbitmq_port,
virtual_host='/',
credentials=credentials,
heartbeat_interval=0
)
)
logger.info("Message server(%s:%s) started." % (LOCAL_CONFIG.rabbitmq_server, LOCAL_CONFIG.rabbitmq_port))
except Exception as e:
logger.error(e)
finally:
lock.release() # 解锁
return cls._instance
def __init__(self, queue):
"""
构造方法
:param queue: 队列名称
"""
self.queue = queue
credentials = pika.PlainCredentials('8lab', '8lab')
self.channel = None
try:
connection = pika.BlockingConnection(
# heartbeat_interval=0表示关闭server的心跳检测, 防止超时关闭
pika.ConnectionParameters(host=LOCAL_CONFIG.rabbitmq_server,
port=LOCAL_CONFIG.rabbitmq_port,
virtual_host='/',
credentials=credentials,
heartbeat_interval=1))
if connection:
print('connected to rabbitMQ successfully.')
self.channel = connection.channel()
self.channel.queue_declare(queue=queue) # 生命channel的队列名称
self.channel.basic_consume(self.callback, queue=self.queue)
else:
print('[8lab][ERROR] init clamav_msg_receiver error ')
logger.error('init clamav_msg_receiver error ')
except Exception as e:
logger.error(
"ProbableAuthenticationError: Cannot connect RabbitMQ server.")
def link_test(request):
"""
测试连接
:param: hostip
:return: 200 测试连接成功 其他 测试连接失败
"""
if request.method == 'POST':
# 接收参数
hostip = request.POST.get('hostip')
# 校验参数
if not hostip:
return JsonResponse({'code': '201', 'message': 'IP不能为空'})
res = re.match(r'^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$',
hostip)
if res is None:
return JsonResponse({'code': '201', 'message': 'IP不合法'})
try:
res = scan.say_hello(hostip)
except Exception as e:
logger.error(str(e))
return JsonResponse({'code': '201', 'message': '请开启本地服务'})
if res:
return JsonResponse({'code': '200', 'message': '测试连接成功'})
return JsonResponse({'code': '201', 'message': '测试连接失败'})
else:
return JsonResponse({'code': '201', 'message': '访问方式出错'})
def send_mail(to_list, sub, content):
"""
发送邮件
:param to_list:
:param sub:
:param content:
:return:
"""
email_server = None
is_send_successful = False
try:
msg = MIMEText(content, _subtype='plain', _charset='gb2312')
msg['Subject'] = sub
msg['From'] = me
msg['To'] = ";".join(to_list)
email_server = smtplib.SMTP()
email_server.connect(mail_host, mail_port)
email_server.login(mail_user, mail_pass)
email_server.sendmail(me, to_list, msg.as_string())
is_send_successful = True
except Exception as e:
logger.error(e)
finally:
if email_server:
email_server.close()
return is_send_successful
def get_waf_logs_timestamp_count(time_interval=10):
"""
按秒数统计日志,即每秒出现多少条日志
:param time_interval: 时间过滤,最近time_interval秒内
:return:
"""
result_dict = {"data": []}
es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port) # es实例
now_interval, now = get_range_of_last_interval(time_interval)
try:
body = {
"query": {
"range": {
"@timestamp": {
"gte": now_interval,
"lte": now
}
},
}
}
results = es.search(index=WAF_INDEX,
doc_type="wafLog",
body=body,
ignore_unavailable=True) # 访问一次ES
result_dict["data"] = results["hits"]["hits"]
except Exception as e:
logger.error(e)
finally:
return result_dict
def publish_confirmed(self, channel, message, message_confirmed_key,
task_id):
"""
发送任务之后, 检验消息是否收到
:param channel: 消息队列名称
:param message: 消息
:param message_confirmed_key: 用于获取确认消息的key
:param task_id: 任务ID
:return:
"""
# 发布任务
self.redis.rdb.publish(channel, message)
# 校验任务是否收到, 时间为3秒钟
for i in range(300):
message = self.redis.rdb.get(message_confirmed_key)
# 任务已被确认收到
if message and message == 'confirmed':
msg = 'GET Task %s confirmed.' % str(task_id)
logger.info(msg)
return True, msg
time.sleep(0.01)
msg = 'Task %s unconfirmed' % str(task_id)
logger.error(msg)
return False, msg
def __enter__(self):
try:
self.transport.open()
logger.info("Nisa RPC Client connected.")
return self.client
except Exception as e:
logger.error(e)
def waf_home(request):
"""
返回waf首页所需的4种数据
:param request:
:return:
"""
return_dict = {}
try:
# 返回近7天每天的每种攻击类型的统计,和每种类型的总数统计
week, total = utils_waf.get_waf_log_aggregations_week()
return_dict["week"] = week
return_dict["total"] = total
# return_dict["total"] = {"web-attack": 10, "sensitive-data-tracking": 0,
# "identification-error": 50, "dos-attack": 15,
# "http-defense": 30}
# 返回近10天每天的日志总条数统计
return_dict["days"] = utils_waf.get_waf_log_aggregations_days()
# 返回攻击源城市统计
return_dict["city"] = utils_waf.get_waf_log_aggregations_city()
except Exception as e:
logger.error(str(e))
return HttpResponse(json.dumps(return_dict))
def _get_watcherlab_statics():
"""
获取一定时间内的态势感知数据
:param time: 分钟数, 时间长度
:param limit: 分页大小
:return: 该时间段内的攻击信息
"""
result = {}
try:
es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port)
# 首先在es中搜索各索引全部的wafLog记录
body = {
"query": {},
}
_result = es.search(index='watcherlab*',
body=body,
ignore_unavailable=True) # 从es中读取
total = _result['hits']['total']
result['intercepted'] = total
except Exception as e:
result['intercepted'] = 1
logger.error(e)
logger.error(traceback.format_exc())
finally:
return result
def get_attack_pct(request):
"""
获取五种攻击百分比
:param request:
:return:
"""
return_dict = {}
nums = 0
try:
# 返回近7天每天的每种攻击类型的统计,和每种类型的总数统计
week, total = utils_waf.get_waf_log_aggregations_week()
for item in total.keys():
nums += total[item]
nums = nums if nums else 1
return_dict['web-attack'] = round(total['web-attack'] / nums, 4)
return_dict['sensitive-data-tracking'] = round(
total['sensitive-data-tracking'] / nums, 4)
return_dict['identification-error'] = round(
total['identification-error'] / nums, 4)
return_dict['dos-attack'] = round(total['dos-attack'] / nums, 4)
return_dict['http-defense'] = round(total['http-defense'] / nums, 4)
except Exception as e:
logger.error(e)
return HttpResponse(json.dumps(return_dict))
def invoke_rpc_repair(contents):
"""
调用RPC进行修复
:param contents:
:return: 执行状态
"""
try:
"""
result = TrustLog.objects.filter(id=log_id, state=0)
if not result:
return False
confirm_item = result[0]
"""
with BlackBoxRPCClient(BLACKBOX_RPC_IP, BLACKB0X_RPC_PORT) as client:
for tmp_key in contents.keys():
content = contents[tmp_key]
for tmp in content:
repair_pack = RepairMessage(tmp['content'], tmp['host'],
tmp['ip'])
logger.debug(vars(repair_pack))
result = client.command(json.dumps(vars(repair_pack)))
logger.debug(result)
# 更新trustlog表中的state值
error_log = TrustLog.objects.get(id=tmp['id'])
error_log.state = 1
error_log.save()
logger.info("Invoking RPC to repair.")
return True
except Exception as e:
logger.error(e)
return False
def get_db_by_host(ip_address):
"""
根据主机IP获得对应数据库信息
:param ip_address: 主机IP
:return: 该主机上保存的数据库
"""
nodes = []
# conn, cursor = get_conn_cursor() 88888
try:
# sql = 'SELECT db_name, db_ip, db_type, db_size FROM client_info WHERE db_ip=%s'
# sql = 'SELECT * FROM client_info WHERE db_ip=%s' 88888
# result = mysql_base_api.sql_execute(conn, cursor, sql, (ip_address,)) 88888
result = Database.objects.filter(db_ip=ip_address).values()
if result and len(result) > 0:
for row in result:
nodes.append({
'dbname': row['db_name'],
'hostip': row['db_ip'],
'dbtype': row['db_type'],
'dbsize': row['db_size'],
'dbport': row['db_port']
})
except Exception as e:
logger.error(e)
# finally: 88888
# mysql_base_api.sql_close(conn, cursor) 88888
return nodes
def update_virus_book():
try:
trustlogs = TrustLog.objects.filter(level='ERROR', state=0)
if trustlogs:
create_list = []
for item in trustlogs:
for filename in item.filename.split("\n"):
length = len('file_error_path:')
path_index = filename.find('file_error_path:') + length
hash_index = filename.find('file_error_hash:') + length
file_hash = filename[path_index:filename.
find('file_error_hash:')].strip()
file_path = filename[hash_index:].strip()
virus = Virusbook(resource=file_hash,
responseCode=1,
fileName=file_path,
host=item['host'],
ip=item['ip'])
create_list.append(virus)
TrustLog.objects.bulk_create(create_list)
else:
Virusbook.objects.raw("truncate virusbook")
except Exception as e:
logger.error(e)
def del_white_one(request):
"""
白名单删除一条记录
:param request: ids
:return:
"""
response = Response()
try:
# 接收参数
_id = request.POST['ids']
# 根据id获取白名单对象 进行删除
target = WhiteList.objects.get(id=_id)
ret = {
"filedata": target.file_data,
"ip": target.ip,
"filerouter": target.file_router,
"id": target.id,
"clientname": target.client_name
}
zmq_class.send_reset(ret)
target.delete()
except Exception as e:
logger.error(str(e))
response.setcode('ERROR')
response.adddata("message", str(e))
return HttpResponse(json.dumps(response.__dict__))
def set_clamav_config(host_ip, config):
"""
保存clamascan配置的参数
:param config: 配置参数
:return: 是否保存成功
"""
if host_ip is None or config is None:
return False
execute_now(config)
# 若相应主机IP的记录存在则更新,否则更新
sql = "INSERT INTO system_config(host_ip, type, config) VALUES(%s, %s, %s) " \
"ON DUPLICATE KEY UPDATE config=%s"
conn = None
cursor = None
try:
conn, cursor = mysql_base_api.sql_init(CONFIG.mysql_host,
CONFIG.mysql_user,
CONFIG.mysql_pass,
CONFIG.mysql_database)
config_str = json.dumps(config)
mysql_base_api.sql_execute(conn, cursor, sql,
[host_ip, TYPE, config_str, config_str])
return True
except Exception as e:
logger.error(e)
finally:
if conn is not None and cursor is not None:
mysql_base_api.sql_close(conn, cursor)
return False
def del_white(request):
"""
白名单删除多条记录
:param request: ids = WL247153WL247177
:return:
"""
response = Response()
try:
# 这里split之后的list中，第一个元素会是一个空格，因此要从第二个元素开始取值
id_list = request.POST.get("ids", "").split("WL")[1:]
# 获取id在id_list范围里的所有白名单对象 进行删除
targets = WhiteList.objects.filter(id__in=id_list)
for target in targets:
ret = {
"filedata": target.file_data,
"ip": target.ip,
"filerouter": target.file_router,
"id": target.id,
"clientname": target.client_name
}
zmq_class.send_reset(ret)
targets.delete()
except Exception as e:
logger.error(str(e))
response.setcode('ERROR')
response.adddata("message", str(e))
return HttpResponse(json.dumps(response.__dict__))
def update_warninglist(request):
"""
更新 警报名单 包括删除 启用 停用
:param request:
:return:
"""
response_result = {}
try:
ids = request.POST.get("data")
action_type = request.POST.get("type")
_ids = ids[:-1].split('#')
logger.debug(_ids)
if action_type == "0": # 删除
WarningList.objects.filter(id__in=_ids).delete()
elif action_type == "1": # 停用
WarningList.objects.filter(id__in=_ids).update(enabled=0)
elif action_type == "2": # 启用
WarningList.objects.filter(id__in=_ids).update(enabled=1)
clear_mail_to_list_cache() # 清除redis缓存
response_result = {'code': '1'}
except Exception as e:
response_result = {'code': '0'}
logger.error(e)
finally:
return HttpResponse(json.dumps(response_result))
def get_scanport_config(host_ip):
"""
获取端口扫描的配置
:return: 扫描端口相关配置
"""
# conn = None
# cursor = None
default = "{}"
try:
# 130-135 88888
# conn, cursor = mysql_base_api.sql_init(
# CONFIG.mysql_host, CONFIG.mysql_user,
# CONFIG.mysql_pass, CONFIG.mysql_database
# )
# sql = "SELECT config FROM system_config WHERE host_ip=%s AND type=%s"
# result = mysql_base_api.select_one_row(cursor, sql, [host_ip, TYPE])
result = SystemConfig.objects.filter(host_ip=host_ip, type=TYPE).values('config')[0]
if result is None:
return default
return result.get('config', default)
except Exception as e:
logger.error(e)