def extract_data_from_mail(m): if not m: return None type = m.get_header("content-type", "") body = m.body compressing = m.get_header("compressing", "") if "error" == type: raise RuntimeError(text(body)) if compressing: if "gzip" == compressing: body = gzip.zlib.decompress(str(body)) else: raise RuntimeError("unsupported compressing method") if "event" == type: text_body = text(body) event = Event.from_json(text_body) return event elif "event[]" == type: events = json.loads(text(body)) events = [Event.from_dict(_) for _ in events] return events else: raise RuntimeError( "this body type ({}) is not supported yet".format(type))
def extract_regist_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "/user/regist" not in httpmsg.uri: return result = "F" body = httpmsg.resp_body or "" if body: try: body = get_json_obj(body) code = body["code"] if code == 200: result = "T" except Exception as err: pass properties = extract_common_properties(httpmsg) properties["result"] = result properties["register_realname"] = "" properties["register_channel"] = "" properties["email"] = "" properties["user_name"] = extract_value_from_body(r_mobile_pattern, httpmsg.req_body) properties["password"] = "" properties["captcha"] = "" properties["register_verification_token"] = "" properties["register_verification_token_type"] = "" return Event("nebula", "ACCOUNT_REGISTRATION", "", millis_now(), properties)
def extract_login_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "users/login" not in httpmsg.uri: return properties = extract_common_properties(httpmsg) result = "F" body = httpmsg.resp_body or "" if body: try: body = get_json_obj(body) succ = body["code"] if succ == 200: result = "T" except Exception as err: pass body = httpmsg.req_body or "" login_name = extract_value_from_body(l_name_pattern, httpmsg.req_body) properties["result"] = result properties["password"] = get_md5( extract_value_from_body(l_passwd_pattern, httpmsg.req_body)) properties["user_name"] = login_name properties["captcha"] = "" properties["remember_me"] = "F" properties["login_channel"] = "pc" properties["login_verification_type"] = "password" properties["uid"] = login_name return Event("nebula", "ACCOUNT_LOGIN", "", millis_now(), properties)
def get_latest_statistic(key, key_type, var_list, subkeys=None): data = { "app": "nebula", "count": 100, "var_list": var_list, "key_type": key_type } if subkeys: data['subkeys'] = subkeys logger.debug( DEBUG_PREFIX + u"获取最近的事件们key:%s, type:%s, key_type:%s, 变量列表:%s", key, type(key), key_type, var_list) request = Event("__all__", "keystatquery_request", key, millis_now(), data) response = statQueryClient.send(request, key, block=False, timeout=5) if response[0]: if isinstance(response[1], list): result = dict() for r in response[1]: logger.debug(DEBUG_PREFIX + "返回的一个event:%s", r) result = dict_merge(result, r.property_values.get("result", {})) else: result = response[1].property_values.get("result", {}) logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response) result = dict() return result
def extract_login_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if httpmsg.uri != "/member/login": return properties = extract_common_properties(httpmsg) if "err" in httpmsg.resp_body: result = "F" else: result = "T" properties["result"] = result body = httpmsg.req_body or "" properties["password"] = get_md5( get_md5( get_md5( get_md5( get_md5(extract_value_from_body(l_passwd_pattern, body)))))) properties["login_name"] = extract_value_from_body(l_name_pattern, body) properties["login_type"] = "pc" properties["auth_msg"] = "" properties["autologin"] = False properties["captcha"] = "" return Event("nebula", "loginlog", "", millis_now(), properties)
def events_from_dynamic(self, result, http_msg): # 动态脚本判断获取 event # 得到一个列表, 然后循环插入 # 扫描lib文件夹的所有 (root, all_py) = self.py_from_address() properties = extract_common_properties(http_msg) all_py = delete_pyc(all_py) for f in all_py: # path = "nebula_sniffer.nebula_sniffer.customparsers.lib." + f[0: -3] try: path = "nebula_sniffer.customparsers.lib." + f[0:-3] e = importlib.import_module(path) p = json.dumps(properties) out = e.event(p) out = json.loads(out) for o in out: if o['event_result'] is True: event_name = o['event_name'] properties = o['properties'] e = Event("nebula", event_name, "", millis_now(), properties) result.append(e) else: pass except Exception as f: print('import error', f) return result
def extract_login_log_event(httpmsg): """ Login event extractor """ if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "/checkcode" not in httpmsg.uri: return properties = extract_common_properties(httpmsg) body = httpmsg.req_body or "" result = "F" if httpmsg.status_code == 302 and '''The URL has moved <a href="http://login.passport.9you.com/loginloading.jsp''' \ in httpmsg.resp_body: result = "T" properties["login_result"] = result properties["password"] = get_md5( extract_value_from_body(l_passwd_pattern, body)) properties["login_name"] = extract_value_from_body(l_name_pattern, body) properties["login_type"] = "pc" properties["auth_msg"] = "" properties["autologin"] = False properties["captcha"] = extract_value_from_body(l_captcha_pattern, body) return Event("nebula", "loginlog", "", millis_now(), properties)
def baseline(count, topcount, key_variable, key_dimension, var_list, merge_list, timestamp): global logger logger = init_env("offline.query.baseline") data = dict() data['count'] = count data['topCount'] = topcount data['key_variable'] = [ key_variable, ] data['key_dimension'] = key_dimension data['var_list'] = var_list.split(",") data['merge_list'] = merge_list.split(",") data["timestamp"] = int(timestamp) req = Event("nebula", "offline_baselinekeystatquery", "", millis_now(), data) BaselineClient = babel.get_offline_baseline_query_client() response = BaselineClient.send(req, "", timeout=10) if response[0]: if isinstance(response[1], list): result = dict() for r in response[1]: logger.debug(DEBUG_PREFIX + "返回的一个event:%s", r) result = dict_merge( result, r.property_values.get("result", {}) or dict()) else: result = response[1].property_values.get("result", {}) logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response)
def get_licenseinfo(): global now, licenseinfo # 保存查询结果1分钟 if (curr_timestamp() - now) > 60 or not licenseinfo: try: client = get_client(settings.LicenseInfo_redis, settings.LicenseInfo_rmq) event = Event('nebula_web', 'licenseinfo', '', millis_now(), {}) bbc, bbc_data = client.send(event, '', True, 10) if bbc: licenseinfo['expire'] = bbc_data.property_values.get( 'days', '') licenseinfo['version'] = bbc_data.property_values.get( 'info', '') now = curr_timestamp() return licenseinfo else: return None except Exception as e: logger.error(e) return None else: return licenseinfo
def key_stat(count, key, dimension, timestamp, var_list): global logger logger = init_env("offline.query.keystat") data = dict() data['app'] = 'nebula' data["key"] = key data["count"] = count data["var_list"] = var_list.split(",") data["dimension"] = dimension data["timestamp"] = timestamp logger.debug(DEBUG_PREFIX + u"查询的时间是%s", datetime.fromtimestamp(int(timestamp) / 1000.0)) req = Event("nebula", "offlinekeystatquery", key, millis_now(), data) KeyStatClient = babel.get_offline_keystat_query_client() response = KeyStatClient.send(req, key, block=False, timeout=5) if response[0] and isinstance(response[1], list): if isinstance(response[1], list): result = dict() for r in response[1]: logger.debug(DEBUG_PREFIX + "返回的一个event:%s", r) result = dict_merge( result, r.property_values.get("result", {}) or dict()) else: result = response[1].property_values.get("result", {}) logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response)
def get_latest_incident(var_list, key='', key_variable='', count=20, page=0): data = dict() data['app'] = 'nebula' data['count'] = count data['page'] = page if key: data['key'] = key if key_variable: data['key_variable'] = key_variable data['var_list'] = var_list request = Event("nebula_web", "incidentquery", key, millis_now(), data) response = Incident_Query_Client.send(request, key, 10) if response[0] and isinstance(response[1], list): values = [event.property_values for event in response[1]] result = dict() for value in values: result = dict_merge(result, value) logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response) result = dict() return result
def anwser(event): logger.debug(DEBUG_PREFIX+"事件 %s 接收的时间: %s", event, datetime.now()) if not event: # 什么时候rpc的event为空? return key = event.key if_all_key = event.property_values['if_all_key'] key_type = event.property_values['key_type'] fromtime = event.property_values['fromtime'] endtime = event.property_values['endtime'] var_names = event.property_values['var_names'] if if_all_key: ret = get_all_statistic(key_type, fromtime, endtime, var_names) else: ret = get_statistic(key, key_type, fromtime, endtime, var_names) logger.debug(DEBUG_PREFIX+u"获取返回的数据是:%s", ret) temp_dict = dict() for var_name, stat in ret.iteritems(): if isinstance(stat, set): temp_dict[var_name] = list(stat) else: temp_dict[var_name] = stat response = Event("__all__", "offline_stat_query_response", key, millis_now(), {'result':temp_dict}) logger.debug(DEBUG_PREFIX+u"rpc server 返回的数据是:%s", response) return response
def get_profile_crawler_risk(current_day, start_day, end_day): """ new in 2.11 不支持polling. Return: (sucess, dict or string) False, error message(string) True, data(any type) """ bn = "ProfileCrawlerRiskClient" # babel request property_values = { 'current_day': current_day, 'start_day': start_day, 'end_day': end_day } event = Event('nebula_web', 'profile_crawler_risk', '', millis_now(), property_values) success, res = ProfileCrawlerRiskClient.send(event, '', True, 10) # babel request fail if not success: msg = u"%s Babel request fail, event: %s" % (bn, event) logger.error(msg) return False, msg # bad request _ = res.property_values if _.has_key("status"): msg = u"Bad %s response event: %s, status: %s, msg:%s " % (\ bn, event, _.get("status"), _.get("msg")) logger.error(msg) return False, msg return True, _
def parse_event(self, src_event, http_msg): """ :param src_event: 原始事件 :param http_msg: 关联的http数据 :return: 生成的事件;如果条件不满足,返回None """ src_event_properties = src_event.property_values http_data_context = HttpDataContext() http_data_context.from_http_msg(http_msg) for c in self.pre_conditions: if not c.eval(src_event_properties, http_msg, http_data_context): return None dst_properties = dict() for m in self.mappings: name, value = m.map(src_event_properties, http_msg, http_data_context) if name is not None: dst_properties[name] = value for field_name, default_value in self.dst_field_default_values.items(): if field_name not in dst_properties: dst_properties[field_name] = default_value return Event("nebula", self.dst_event_name, dst_properties["c_ip"], dst_properties["timestamp"], dst_properties)
def extract_app_regist_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "/api/c1_register" not in httpmsg.uri: return properties = extract_common_properties(httpmsg) result = "F" try: j = json.loads(httpmsg.resp_body) if j["err"] is None: result = "T" except: pass properties["result"] = result body = httpmsg.req_body or "" properties["email"] = "" properties["regist_name"] = extract_value_from_body( app_r_name_pattern, body) properties["mobile"] = extract_value_from_body(app_r_name_pattern, body) properties["password"] = get_md5( get_md5( get_md5( get_md5( get_md5(extract_value_from_body(l_passwd_pattern, body)))))) properties["captcha"] = "" return Event("nebula", "registlog", "", millis_now(), properties)
def extract_login_log_event(httpmsg): """ Login event extractor """ if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if httpmsg.uri != "/m/login.html": return properties = extract_common_properties(httpmsg) body = httpmsg.resp_body or "" result = get_result(body) properties["login_result"] = result properties["password"] = get_md5( extract_value_from_body(l_passwd_pattern, body)) properties["login_name"] = extract_value_from_body(l_name_pattern, body) properties["login_type"] = "pc" properties["auth_msg"] = "" properties["autologin"] = False properties["captcha"] = extract_value_from_body(l_captcha_pattern, body) return Event("nebula", "loginlog", "", millis_now(), properties)
def get_latest_events(key, key_type, fromtime=None, size=None, event_id=None, only_count=False): logger.debug(DEBUG_PREFIX + u"获取最近的事件们key:%s, type:%s, key_type:%s", key, type(key), key_type) prop_dict = dict(key_type=key_type, only_count=only_count) if fromtime: prop_dict['fromtime'] = fromtime if size: prop_dict['size'] = size if event_id: prop_dict['eventid'] = event_id request = Event("__all__", "eventquery_request", key, millis_now(), prop_dict) response = eventQueryClient.send(request, key, block=False, timeout=5) if response[0]: value = response[1].property_values.get("result") logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, value) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response) return cached_data
def extract_regist_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "pc/mobileregister.action" not in httpmsg.uri: return result = "F" body = httpmsg.resp_body or "" if body: try: body = get_json_obj(body) code = body["code"] if code == 1000: result = "T" except Exception as err: pass properties = extract_common_properties(httpmsg) properties["regist_result"] = result properties["email"] = "" properties["regist_name"] = extract_value_from_body(r_mobile_pattern, body) properties["password"] = "" properties["captcha"] = "" return Event("nebula", "registlog", "", millis_now(), properties)
def get_latest_baseline_statistic(key_variable, var_list, merge_list=None, count=100, topcount=1): data = dict() data['app'] = 'nebula' data['count'] = count data['topcount'] = topcount data['key_variable'] = key_variable data['var_list'] = var_list if merge_list: data['merge_list'] = merge_list request = Event("nebula_web", "baselinekeystatquery", '', millis_now(), data) response = Baseline_Query_client.send(request, '', 7) if response[0] and isinstance(response[1], list): values = [event.property_values for event in response[1]] result = dict() for value in values: result = dict_merge(result, value) logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response) result = dict() return result
def extract_password_modify_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if 'ucenter/repassword.action' not in httpmsg.uri: return properties = extract_common_properties(httpmsg) result = "F" body = httpmsg.resp_body or "" if body: try: body = get_json_obj(body) code = body["code"] if code == 1000: result = "T" except Exception as err: pass properties["result"] = result properties["old_password"] = get_md5(extract_value_from_body(pwmodify_oldpwd_pattern, httpmsg.req_body)) properties["new_password"] = get_md5(extract_value_from_body(pwmodify_newpwd_pattern, httpmsg.req_body)) properties["username"] = extract_value_from_body(pwmodify_username_pattern, httpmsg.req_body) return Event("nebula", "password_modify", httpmsg.source_ip, millis_now(), properties)
def extract_password_reset_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if "getpassword.aspx" not in httpmsg.uri_stem: return properties = extract_common_properties(httpmsg) result = "F" for header in httpmsg.resp_headers.itervalues(): if "logined=1; expires=" in header: result = "T" properties["result"] = result properties["mobile"] = extract_value_from_body(pwreset_mobile_pattern, httpmsg.req_body) properties["captcha"] = extract_value_from_body(pwreset_captcha1_pattern, httpmsg.req_body) or \ extract_value_from_body(pwreset_captcha2_pattern, httpmsg.req_body) properties["auth_msg"] = extract_value_from_body(pwreset_auth_pattern, httpmsg.req_body) properties["new_password"] = get_md5( extract_value_from_body(pwreset_newpw_pattern, httpmsg.req_body)) step = 1 if "btn_step2" in httpmsg.req_body: step = 2 elif "btn_step3" in httpmsg.req_body: step = 3 properties["step"] = step return Event("nebula", "password_reset", httpmsg.source_ip, millis_now(), properties)
def extract_http_log_text_msg(textmsg): if not textmsg: return None t = textmsg.text j = json.loads(t) m = j["message"] remote_ip, _, _, _, _, log = m.split(" ", 5) parts = extract_nginx_log_parts(log) user_agent = parts[5] or "" referer = parts[4] or "" method, uri = parts[0].split(" ")[:2] uri = uri.lower() if parts[2]: status = int(parts[2]) else: status = 0 host = parts[1] if parts[7]: server_ip, server_port = parts[7].split(":") else: server_ip, server_port = "", "0" if parts[3]: s_bytes = int(parts[3]) else: s_bytes = 0 # get full uri if "?" not in uri: uri_stem = uri uri_query = "" else: uri_stem, uri_query = uri.split("?", 1) properties = dict() properties["c_ip"] = remote_ip properties["c_ipc"] = ".".join(remote_ip.split(".")[:3]) properties["c_port"] = 0 properties["uri_stem"] = uri_stem properties["uri_query"] = uri_query properties["host"] = host properties["useragent"] = user_agent properties["status"] = status properties["referer"] = referer properties["c_body"] = "" properties["c_bytes"] = 0 properties["s_ip"] = server_ip properties["s_ipc"] = ".".join(server_ip.split(".")[:3]) properties["s_port"] = int(server_port) properties["s_body"] = "" properties["s_bytes"] = s_bytes properties["cookie"] = "" properties["method"] = method return Event("nebula", "httplog", "", millis_now(), properties)
def fetch_logquery_data(lq_id, page, page_count): # fetch success logquery job's data via babel bn = "Logquery fetch" prop = dict(id=lq_id, action_type="fetch", page=page, page_count=page_count) event = Event("__all__", "logquery", "", millis_now(), prop) return send_event(event, LogQueryClient, bn)
def extract_http_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return properties = extract_common_properties(httpmsg) # remove parent id. the func-order seem terrible now. properties["pid"] = "0" * 24 event_name = "HTTP_STATIC" if properties["is_static"] else "HTTP_DYNAMIC" return Event("nebula", event_name, "", millis_now(), properties)
def get_offline_key_stat(keys, dimension, timestamp, var_list): """ 获取离线slot变量数据 :return: """ data = dict() data['app'] = 'nebula' data["keys"] = keys if isinstance(var_list, list): data["var_list"] = var_list elif isinstance(var_list, (str, unicode)): data["var_list"] = var_list.split(",") else: return dict() data["dimension"] = dimension data["timestamp"] = timestamp if not keys: top = True else: top = False if top: data['keys'] = ['__GLOBAL__'] data['dimension'] = 'global' req = Event("nebula", "offlinekeystatquery", '__GLOBAL__', millis_now(), data) least_ret = None if dimension != "global": least_ret = 1 response = OfflineKeyStatClient.send(req, '', block=False, timeout=5, least_ret=least_ret) if response[0]: if isinstance(response[1], list): result = dict() for r in response[1]: logger.debug(DEBUG_PREFIX + "返回的一个event:%s", r) result = dict_merge( result, r.property_values.get("result", {}) or dict()) else: result = response[1].property_values.get("result", {}) if top and result: result = result['__GLOBAL__'] logger.debug(DEBUG_PREFIX + "有返回的结果是:%s, 返回的结果是%s", response, result) return result else: logger.debug(DEBUG_PREFIX + "当前没有事件..., 返回的是%s", response)
def get_page_risk(current_day, pages): # 查询profile账号来源分析 property_values = {'current_day': current_day, 'pages': pages} client = get_profile_page_risk_client() event = Event('nebula_web', 'profile_page_risk', '', millis_now(), property_values) # client发送event,如果RPC正常返回,则返回RPC server返回数据 bbc, bbc_data = client.send(event, '', True, 10) property_values = bbc_data.property_values if bbc else False return property_values
def extract_regist_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "POST": return if httpmsg.uri not in {"/m/register.html", "/m/sendregistercode.html"}: return key = (httpmsg.source_ip, httpmsg.source_port, httpmsg.dest_ip, httpmsg.dest_port) regist_name = "" password = "" captcha = "" regist_result = "F" if httpmsg.uri == "/m/register.html": req_body = httpmsg.req_body or "" resp_body = httpmsg.resp_body or "" regist_name = extract_value_from_body(r_name_pattern, req_body) password = get_md5(extract_value_from_body(r_passwd_pattern, req_body)) captcha = extract_value_from_body(r_captcha_pattern, req_body) result = get_result(resp_body) if result: # success, go to the cache, and return for further process regist_cache[key] = { "regist_name": regist_name, "password": password, "captcha": captcha } return else: regist_result = "F" else: # should be the third step resp_body = httpmsg.resp_body or "" result = get_result(resp_body) if result: regist_result = "T" else: regist_result = "F" data = regist_cache.get(key, {}) if not data: regist_result = "F" regist_name = data.get("regist_name", "") password = data.get("password", "") captcha = data.get("captcha", "") properties = extract_common_properties(httpmsg) properties["regist_result"] = regist_result properties["email"] = "" properties["regist_name"] = regist_name properties["password"] = password properties["captcha"] = captcha return Event("nebula", "registlog", "", millis_now(), properties)
def extract_auth_msg_send_log_event(httpmsg): if not isinstance(httpmsg, HttpMsg): return if httpmsg.method != "GET": return if "api/sms/send_sms" not in httpmsg.uri: return properties = extract_common_properties(httpmsg) result = "T" properties["result"] = result properties["mobile"] = extract_value_from_body(a_name_pattern, httpmsg.uri_query) return Event("nebula", "auth_msg_send", "", millis_now(), properties)
def get_license_info(): try: event = Event('nebula_web', 'licenseinfo', '', millis_now(), {}) bbc, bbc_data = licenseInfoClient.send(event, '', True, 5) if bbc: licenseinfo = dict() licenseinfo['expire'] = bbc_data.property_values.get('days', '') licenseinfo['version'] = bbc_data.property_values.get('info', '') return licenseinfo else: return None except Exception as e: logger.error(e) return None
def get_profile_data(key, key_type, variables): # 初始化profilequery RPC client property_values = { 'profile_key_value': key, 'profile_key_type': key_type, 'variables': variables } event = Event('nebula_web', 'profile_query', '', millis_now(), property_values) # client发送event,如果RPC正常返回,则返回RPC server返回数据 bbc, bbc_data = ProfileQueryClient.send(event, '', True, 10) profile_values = bbc_data.property_values if bbc else False return profile_values