def set_scanport_config(host_ip, config): """ 设置端口扫描的配置 :return: 是否设置成功 """ # 92-95 88888 # sql = "INSERT INTO system_config(host_ip, type, config) VALUES(%s, " \ # "%s, %s) ON DUPLICATE KEY UPDATE config=%s" # conn = None # cursor = None try: # 99-103 88888 # conn, cursor = mysql_base_api.sql_init( # CONFIG.mysql_host, CONFIG.mysql_user, # CONFIG.mysql_pass, CONFIG.mysql_database # ) # mysql_base_api.sql_execute(conn, cursor, sql, [host_ip, TYPE, config, config]) SystemConfig.objects.update_or_create(host_ip=host_ip, type=TYPE, defaults={'config': config}) # 相当于先去 get(host_ip=host_ip, type=TYPE) 如果存在更新 config # 不存在 create(host_ip=host_ip, type=TYPE, config=config) return True except Exception as e: logger.error(e) return False
def update_op_host(op_type, mac, exe_path): """ 更新可信运维主机 :param op_type: :param mac: :param exe_path: :return: """ result = {"status": 'error'} try: if op_type == "create": OperationHost.objects.create(id=None, mac_address=mac, exe_path=exe_path) result['status'] = 'success' elif op_type == "delete": objs = list(OperationHost.objects.filter(mac_address=mac)) if len(objs) > 0: objs[0].delete() result['status'] = 'success' except Exception as e: result['status'] = 'error' logger.error(e) return result
def get_scan_config(host_ip): """ 获取制定主机的扫描配置 :param host_ip: 主机IP, :return: 配置信息 """ conn = None cursor = None default = "{}" try: conn, cursor = mysql_base_api.sql_init(CONFIG.mysql_host, CONFIG.mysql_user, CONFIG.mysql_pass, CONFIG.mysql_database) sql = "SELECT config FROM system_config WHERE host_ip=%s and type=%s" result = mysql_base_api.select_one_row(cursor, sql, [host_ip, TYPE]) if result is None: return default return result.get('config', default) except Exception as e: logger.error(e) finally: if conn is not None and cursor is not None: mysql_base_api.sql_close(conn, cursor)
def get_watcherlab_daily(n): """ 获取每天总的威胁情报数量 :return: 返回近n天攻击信息 """ result = OrderedDict() try: es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port) # 对每天情报数量进行聚合 body = { "size": 0, "aggs": { "date": { "date_histogram": { "field": "@timestamp", "interval": "day", "format": "yyyy-MM-dd" } } } } _result = es.search(index='watcherlab-*', body=body, ignore_unavailable=True) # 从es中读取 _result = _result['aggregations']['date']["buckets"] items = _result[-n:] if len(_result) > n else _result for item in items: result[item["key_as_string"]] = item["doc_count"] except Exception as e: logger.error(e) logger.error(traceback.format_exc()) finally: return result
def virus_init(): # 威胁分析结果存入virusbook表中 update_virus_book() path_value = [] state_code = 2 # 判断是否有威胁的标志 white_list = [] for item in Virusbook.objects.all(): if item.fileName: path_value.append(item.fileName) if item.isWhite == 0: state_code = 1 else: # 将微步认为是白名单的信息添加在表whitelist中 white_list_item = WhiteList(client_name=item.host, ip=item.ip, file_router=item.fileName, file_data=item.resource) white_list.append(white_list_item) # 将微步认为是白名单信息的记录通过rpc发送给blackbox save_whitelog(model_to_dict(white_list_item)) # 并在trustlog中设置level为DEBUG TrustLog.objects.filter( host=item.host, ip=item.ip, filename=item.fileName).update(level='DEBUG') # 在virusbook中删除该条记录 Virusbook.objects.filter(id=item.id).delete() WhiteList.objects.bulk_create(white_list) path_value.append(state_code) (status, output) = subprocess.getstatusoutput("/8lab/blackbox -w") if status != 0: logger.error("[BLACKBOX][ERROR] blackbox reset white list failed!") return path_value
def handle_get_waf_log(results, attack_type): """ 处理wafLog,记录waf攻击的攻击源,攻击目标,攻击时间,攻击工具,攻击类型信息 :param results: :param attack_type: 攻击类型 :return: """ logs = [] time_delta = datetime.timedelta(hours=8) for hit in results: try: log = { "inter_ip": hit['_source']['type'], # 攻击目标ip "inter_time": (datetime.datetime.strptime( hit["_source"]["@timestamp"], "%Y-%m-%dT%H:%M:%S.%fZ") + time_delta).strftime('%Y-%m-%d %H:%M:%S'), "inter_source": hit['_source'].get('XForwardedFor', ''), # 攻击源ip "inter_tool": hit['_source'].get('Referer', ''), # 攻击来源 "defend_type": attack_type } # 防御类型 logs.append(log) except Exception as e: logger.error(str(e)) logs.sort(key=lambda x: x["inter_time"], reverse=True) # 之前已经经过反序,这一步仍检测一次是否有乱序 return logs
def get_details_time(time=5, limit=100): """ 获取一定时间内的攻击信息 :param time: 秒, 时间长度 :param limit: 分页大小 :return: 该时间段内的攻击信息 """ conn = None cursor = None try: conn = get_connection() cursor = conn.cursor(MySQLdb.cursors.DictCursor) sql = "SELECT source_ip, des_ip, attack_type, occur_time " \ "FROM attack_ip WHERE occur_time " \ "BETWEEN date_sub(now(), INTERVAL %d MINUTE) AND now() LIMIT %d" \ % (time, limit) # logger.info(sql) cursor.execute(sql) query_result = cursor.fetchall() return query_result except Exception as e: logger.error(e) finally: cursor.close() conn.close()
def black_list_status(request): """ 返回黑名单(恶意行为) :param request: 参数1: shenji_ip 参数2: type 3为黑名单行为 :return: """ # 接收参数 audit_ip = request.POST.get("shenji_ip", "#") _type = request.POST.get("type") return_dict = {} # 返回的字典 hosts = jc.db_ip_list ips = [] for host in hosts: ips.append(host['ip']) # audit_ip 获取不到就用 ips 里的第一个ip audit_ip = ips[0] if audit_ip == "#" else audit_ip try: # 把黑名单行为(恶意行为)放入到 black_action中 black_action = list(WhiteUserAction.objects.filter( ip=audit_ip, type=_type).order_by('-id').values('sid', 'value')) # id降序查询 结果 [{'sid': sid1, 'value': value1}, {'sid': sid2, 'value': value2}] return_dict['action'] = black_action except Exception as e: logger.error(e) return HttpResponse(json.dumps(return_dict))
def send_message(content): """ 发送短信 :return: """ req = top.api.AlibabaAliqinFcSmsNumSendRequest() req.set_app_info( top.appinfo("23827310", "24dbb9f199ea5fbc2826e4f2662c15df")) req.extend = "" req.sms_type = "normal" req.sms_free_sign_name = "八分量持续免疫系统" if content == "": req.sms_param = "" req.sms_template_code = "SMS_67765123" else: req.sms_param = {'errorinfo': content} req.sms_template_code = "SMS_67715193" req.rec_num = "13488690980,17610000528" try: req.getResponse() except Exception as e: if hasattr(e, "submsg"): logger.error("短信发送失败:" + e.submsg) else: logger.error(e)
def white_list_status(request): """ 返回白名单状态 :param request: shenji_ip :return: """ return_dict = {} # 返回的字典 hosts = jc.db_ip_list ips = [] for host in hosts: ips.append(host['ip']) audit_ip = request.POST.get("shenji_ip", "#") # 获取ip # db_type = request.POST.get("db_type").lower() # 获取数据库类型 audit_ip = ips[0] if audit_ip == "#" else audit_ip # 获取数据库的添加的用户白名单 和 数据库行为白名单 user_action_query = WhiteUserAction.objects.filter(ip=audit_ip).order_by('-id') # 从white user表中查询所有的user_name,从white action表中查询所有的 action_name # 把用户白名单放入到white_user中,把用户行为白名单放入到white_action中 # 当type=1时 返回数据 ['value1', 'value2', 'value3'] # 当type=2时 返回数据 [{'sid': sid1, 'value': value1}, {'sid': sid2, 'value': value2}] try: # flat 为True,它表示返回的结果为单个值而不是元组 ['value1', 'value2', 'value3'] return_dict['user'] = list(user_action_query.objects.filter( type=1).values_list('value', flat=True)) return_dict['action'] = list(user_action_query.objects.filter( type=2).values('sid', 'value')) except Exception as e: logger.error(e) return HttpResponse(json.dumps(return_dict))
def delete_rules(request): """ 黑名单行为删除(恶意行为) :param request: 参数1: values 参数2: ip 审计页面-当前ip :return: """ response_result = {'code': '201'} # 接收参数 values = request.POST.get("values", "")[:-1] # 前台数据 ,由于最后多了一个"#号",所以需要删除 ip = request.POST["ip"] sids = values.split("#") # 数据库删除黑名单行为 try: snort_rpc = SnortRpcClient(ip, 30303) rpc_result = snort_rpc.del_rules(sids) if rpc_result: WhiteUserAction.objects.filter(sid__in=sids, ip=ip).delete() response_result = {'code': '200'} except Exception as e: response_result = {'code': '201'} logger.error(e) return HttpResponse(json.dumps(response_result)) return HttpResponse(json.dumps(response_result))
def clam_scan(self, invoke_ip, conf_str): """ 执行病毒扫描的线程 :param invoke_ip: :return: """ global SCAN_KILLED try: # 更新数据扫描标记位 # MachineList.objects.filter(hostip=invoke_ip).update(is_scan=True) # 开始扫描 result = clamav.clam_scan(invoke_ip, conf_str) if len(result) is not 0 and not SCAN_KILLED: # 将扫描日志和标记位更新 conn, cursor = mysql_base_api.sql_init(LOCAL_CONFIG.mysql_host, LOCAL_CONFIG.mysql_user, LOCAL_CONFIG.mysql_pass, LOCAL_CONFIG.mysql_database, LOCAL_CONFIG.mysql_port) # 初始化数据库 # 从数据库中查询 sql = 'update app_fuzhou_machinelist set is_scan=0,scan_log="%s" where hostip="%s"' \ % (result, invoke_ip) mysql_base_api.sql_execute(conn, cursor, sql, "") # 关闭数据库 mysql_base_api.sql_close(conn, cursor) # MachineList.objects.filter(hostip=invoke_ip).update( # scan_log=str(result), is_scan=False) except Exception as e: logger.error(e) finally: try: MachineList.objects.filter(hostip=invoke_ip).update(is_scan=False) except Exception as e: logger.error(e)
def __new__(cls, *args, **kwargs): lock = Lock() logger.debug("init rabbitMQ instance") if cls._instance is None: try: lock.acquire() # 上锁, 用于保证线程安全性 # 双重锁 if cls._instance is None: cls._instance = super(RabbitMQConnection, cls).__new__(cls) credentials = pika.PlainCredentials('8lab', '8lab') cls._connection = pika.BlockingConnection( # heartbeat_interval=0表示关闭server的心跳检测, 防止超时关闭 pika.ConnectionParameters( host=LOCAL_CONFIG.rabbitmq_server, port=LOCAL_CONFIG.rabbitmq_port, virtual_host='/', credentials=credentials, heartbeat_interval=0 ) ) logger.info("Message server(%s:%s) started." % (LOCAL_CONFIG.rabbitmq_server, LOCAL_CONFIG.rabbitmq_port)) except Exception as e: logger.error(e) finally: lock.release() # 解锁 return cls._instance
def __init__(self, queue): """ 构造方法 :param queue: 队列名称 """ self.queue = queue credentials = pika.PlainCredentials('8lab', '8lab') self.channel = None try: connection = pika.BlockingConnection( # heartbeat_interval=0表示关闭server的心跳检测, 防止超时关闭 pika.ConnectionParameters(host=LOCAL_CONFIG.rabbitmq_server, port=LOCAL_CONFIG.rabbitmq_port, virtual_host='/', credentials=credentials, heartbeat_interval=1)) if connection: print('connected to rabbitMQ successfully.') self.channel = connection.channel() self.channel.queue_declare(queue=queue) # 生命channel的队列名称 self.channel.basic_consume(self.callback, queue=self.queue) else: print('[8lab][ERROR] init clamav_msg_receiver error ') logger.error('init clamav_msg_receiver error ') except Exception as e: logger.error( "ProbableAuthenticationError: Cannot connect RabbitMQ server.")
def link_test(request): """ 测试连接 :param: hostip :return: 200 测试连接成功 其他 测试连接失败 """ if request.method == 'POST': # 接收参数 hostip = request.POST.get('hostip') # 校验参数 if not hostip: return JsonResponse({'code': '201', 'message': 'IP不能为空'}) res = re.match(r'^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$', hostip) if res is None: return JsonResponse({'code': '201', 'message': 'IP不合法'}) try: res = scan.say_hello(hostip) except Exception as e: logger.error(str(e)) return JsonResponse({'code': '201', 'message': '请开启本地服务'}) if res: return JsonResponse({'code': '200', 'message': '测试连接成功'}) return JsonResponse({'code': '201', 'message': '测试连接失败'}) else: return JsonResponse({'code': '201', 'message': '访问方式出错'})
def send_mail(to_list, sub, content): """ 发送邮件 :param to_list: :param sub: :param content: :return: """ email_server = None is_send_successful = False try: msg = MIMEText(content, _subtype='plain', _charset='gb2312') msg['Subject'] = sub msg['From'] = me msg['To'] = ";".join(to_list) email_server = smtplib.SMTP() email_server.connect(mail_host, mail_port) email_server.login(mail_user, mail_pass) email_server.sendmail(me, to_list, msg.as_string()) is_send_successful = True except Exception as e: logger.error(e) finally: if email_server: email_server.close() return is_send_successful
def get_waf_logs_timestamp_count(time_interval=10): """ 按秒数统计日志,即每秒出现多少条日志 :param time_interval: 时间过滤,最近time_interval秒内 :return: """ result_dict = {"data": []} es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port) # es实例 now_interval, now = get_range_of_last_interval(time_interval) try: body = { "query": { "range": { "@timestamp": { "gte": now_interval, "lte": now } }, } } results = es.search(index=WAF_INDEX, doc_type="wafLog", body=body, ignore_unavailable=True) # 访问一次ES result_dict["data"] = results["hits"]["hits"] except Exception as e: logger.error(e) finally: return result_dict
def publish_confirmed(self, channel, message, message_confirmed_key, task_id): """ 发送任务之后, 检验消息是否收到 :param channel: 消息队列名称 :param message: 消息 :param message_confirmed_key: 用于获取确认消息的key :param task_id: 任务ID :return: """ # 发布任务 self.redis.rdb.publish(channel, message) # 校验任务是否收到, 时间为3秒钟 for i in range(300): message = self.redis.rdb.get(message_confirmed_key) # 任务已被确认收到 if message and message == 'confirmed': msg = 'GET Task %s confirmed.' % str(task_id) logger.info(msg) return True, msg time.sleep(0.01) msg = 'Task %s unconfirmed' % str(task_id) logger.error(msg) return False, msg
def __enter__(self): try: self.transport.open() logger.info("Nisa RPC Client connected.") return self.client except Exception as e: logger.error(e)
def waf_home(request): """ 返回waf首页所需的4种数据 :param request: :return: """ return_dict = {} try: # 返回近7天每天的每种攻击类型的统计,和每种类型的总数统计 week, total = utils_waf.get_waf_log_aggregations_week() return_dict["week"] = week return_dict["total"] = total # return_dict["total"] = {"web-attack": 10, "sensitive-data-tracking": 0, # "identification-error": 50, "dos-attack": 15, # "http-defense": 30} # 返回近10天每天的日志总条数统计 return_dict["days"] = utils_waf.get_waf_log_aggregations_days() # 返回攻击源城市统计 return_dict["city"] = utils_waf.get_waf_log_aggregations_city() except Exception as e: logger.error(str(e)) return HttpResponse(json.dumps(return_dict))
def _get_watcherlab_statics(): """ 获取一定时间内的态势感知数据 :param time: 分钟数, 时间长度 :param limit: 分页大小 :return: 该时间段内的攻击信息 """ result = {} try: es = Elasticsearch(LOCAL_CONFIG.es_server_ip_port) # 首先在es中搜索各索引全部的wafLog记录 body = { "query": {}, } _result = es.search(index='watcherlab*', body=body, ignore_unavailable=True) # 从es中读取 total = _result['hits']['total'] result['intercepted'] = total except Exception as e: result['intercepted'] = 1 logger.error(e) logger.error(traceback.format_exc()) finally: return result
def get_attack_pct(request): """ 获取五种攻击百分比 :param request: :return: """ return_dict = {} nums = 0 try: # 返回近7天每天的每种攻击类型的统计,和每种类型的总数统计 week, total = utils_waf.get_waf_log_aggregations_week() for item in total.keys(): nums += total[item] nums = nums if nums else 1 return_dict['web-attack'] = round(total['web-attack'] / nums, 4) return_dict['sensitive-data-tracking'] = round( total['sensitive-data-tracking'] / nums, 4) return_dict['identification-error'] = round( total['identification-error'] / nums, 4) return_dict['dos-attack'] = round(total['dos-attack'] / nums, 4) return_dict['http-defense'] = round(total['http-defense'] / nums, 4) except Exception as e: logger.error(e) return HttpResponse(json.dumps(return_dict))
def invoke_rpc_repair(contents): """ 调用RPC进行修复 :param contents: :return: 执行状态 """ try: """ result = TrustLog.objects.filter(id=log_id, state=0) if not result: return False confirm_item = result[0] """ with BlackBoxRPCClient(BLACKBOX_RPC_IP, BLACKB0X_RPC_PORT) as client: for tmp_key in contents.keys(): content = contents[tmp_key] for tmp in content: repair_pack = RepairMessage(tmp['content'], tmp['host'], tmp['ip']) logger.debug(vars(repair_pack)) result = client.command(json.dumps(vars(repair_pack))) logger.debug(result) # 更新trustlog表中的state值 error_log = TrustLog.objects.get(id=tmp['id']) error_log.state = 1 error_log.save() logger.info("Invoking RPC to repair.") return True except Exception as e: logger.error(e) return False
def get_db_by_host(ip_address): """ 根据主机IP获得对应数据库信息 :param ip_address: 主机IP :return: 该主机上保存的数据库 """ nodes = [] # conn, cursor = get_conn_cursor() 88888 try: # sql = 'SELECT db_name, db_ip, db_type, db_size FROM client_info WHERE db_ip=%s' # sql = 'SELECT * FROM client_info WHERE db_ip=%s' 88888 # result = mysql_base_api.sql_execute(conn, cursor, sql, (ip_address,)) 88888 result = Database.objects.filter(db_ip=ip_address).values() if result and len(result) > 0: for row in result: nodes.append({ 'dbname': row['db_name'], 'hostip': row['db_ip'], 'dbtype': row['db_type'], 'dbsize': row['db_size'], 'dbport': row['db_port'] }) except Exception as e: logger.error(e) # finally: 88888 # mysql_base_api.sql_close(conn, cursor) 88888 return nodes
def update_virus_book(): try: trustlogs = TrustLog.objects.filter(level='ERROR', state=0) if trustlogs: create_list = [] for item in trustlogs: for filename in item.filename.split("\n"): length = len('file_error_path:') path_index = filename.find('file_error_path:') + length hash_index = filename.find('file_error_hash:') + length file_hash = filename[path_index:filename. find('file_error_hash:')].strip() file_path = filename[hash_index:].strip() virus = Virusbook(resource=file_hash, responseCode=1, fileName=file_path, host=item['host'], ip=item['ip']) create_list.append(virus) TrustLog.objects.bulk_create(create_list) else: Virusbook.objects.raw("truncate virusbook") except Exception as e: logger.error(e)
def del_white_one(request): """ 白名单删除一条记录 :param request: ids :return: """ response = Response() try: # 接收参数 _id = request.POST['ids'] # 根据id获取白名单对象 进行删除 target = WhiteList.objects.get(id=_id) ret = { "filedata": target.file_data, "ip": target.ip, "filerouter": target.file_router, "id": target.id, "clientname": target.client_name } zmq_class.send_reset(ret) target.delete() except Exception as e: logger.error(str(e)) response.setcode('ERROR') response.adddata("message", str(e)) return HttpResponse(json.dumps(response.__dict__))
def set_clamav_config(host_ip, config): """ 保存clamascan配置的参数 :param config: 配置参数 :return: 是否保存成功 """ if host_ip is None or config is None: return False execute_now(config) # 若相应主机IP的记录存在则更新,否则更新 sql = "INSERT INTO system_config(host_ip, type, config) VALUES(%s, %s, %s) " \ "ON DUPLICATE KEY UPDATE config=%s" conn = None cursor = None try: conn, cursor = mysql_base_api.sql_init(CONFIG.mysql_host, CONFIG.mysql_user, CONFIG.mysql_pass, CONFIG.mysql_database) config_str = json.dumps(config) mysql_base_api.sql_execute(conn, cursor, sql, [host_ip, TYPE, config_str, config_str]) return True except Exception as e: logger.error(e) finally: if conn is not None and cursor is not None: mysql_base_api.sql_close(conn, cursor) return False
def del_white(request): """ 白名单删除多条记录 :param request: ids = WL247153WL247177 :return: """ response = Response() try: # 这里split之后的list中,第一个元素会是一个空格,因此要从第二个元素开始取值 id_list = request.POST.get("ids", "").split("WL")[1:] # 获取id在id_list范围里的所有白名单对象 进行删除 targets = WhiteList.objects.filter(id__in=id_list) for target in targets: ret = { "filedata": target.file_data, "ip": target.ip, "filerouter": target.file_router, "id": target.id, "clientname": target.client_name } zmq_class.send_reset(ret) targets.delete() except Exception as e: logger.error(str(e)) response.setcode('ERROR') response.adddata("message", str(e)) return HttpResponse(json.dumps(response.__dict__))
def update_warninglist(request): """ 更新 警报名单 包括删除 启用 停用 :param request: :return: """ response_result = {} try: ids = request.POST.get("data") action_type = request.POST.get("type") _ids = ids[:-1].split('#') logger.debug(_ids) if action_type == "0": # 删除 WarningList.objects.filter(id__in=_ids).delete() elif action_type == "1": # 停用 WarningList.objects.filter(id__in=_ids).update(enabled=0) elif action_type == "2": # 启用 WarningList.objects.filter(id__in=_ids).update(enabled=1) clear_mail_to_list_cache() # 清除redis缓存 response_result = {'code': '1'} except Exception as e: response_result = {'code': '0'} logger.error(e) finally: return HttpResponse(json.dumps(response_result))
def get_scanport_config(host_ip): """ 获取端口扫描的配置 :return: 扫描端口相关配置 """ # conn = None # cursor = None default = "{}" try: # 130-135 88888 # conn, cursor = mysql_base_api.sql_init( # CONFIG.mysql_host, CONFIG.mysql_user, # CONFIG.mysql_pass, CONFIG.mysql_database # ) # sql = "SELECT config FROM system_config WHERE host_ip=%s AND type=%s" # result = mysql_base_api.select_one_row(cursor, sql, [host_ip, TYPE]) result = SystemConfig.objects.filter(host_ip=host_ip, type=TYPE).values('config')[0] if result is None: return default return result.get('config', default) except Exception as e: logger.error(e)