def put_nowait(self, obj, routing_key, durable=False): if self.lazy_limit and self.qsize_diff < self.qsize_diff_limit: pass elif self.full(): raise Queue.Full else: self.qsize_diff = 0 with self.lock: self.qsize_diff += 1 RedisCtx.get_instance().redis.rpush(self.name, str(obj))
def init_redis(): from complexconfig.configcontainer import configcontainer host = configcontainer.get_config("sniffer").get_string("sniffer.redis.host") port = configcontainer.get_config("sniffer").get_int("sniffer.redis.port") password = configcontainer.get_config("sniffer").get_string("sniffer.redis.password", "") from threathunter_common.redis.redisctx import RedisCtx RedisCtx.get_instance().host = host RedisCtx.get_instance().port = port RedisCtx.get_instance().password = password print_with_time("successfully init redis[host={},port={},password={}]".format(host, port, "*"*len(password)))
def get_user_from_session(session): if not session: return "" # 1. try load from local cache result = local_cache.get(session) if result == NOT_EXIST_USER: # should searched redis in the past and no value is found return "" elif result: # there is value return result else: # do redis ops pass # 2. try load from redis cache redis_key = "user_for_session_{}".format(session) result = RedisCtx.get_instance().redis.get(redis_key) # populate the cache if result: local_cache[session] = result else: local_cache[session] = NOT_EXIST_USER return result
def start_consuming(self): self.running = True r = RedisCtx.get_instance().redis self.pubsub = r.pubsub() self.pubsub.subscribe(self.name) latch = CountDownLatch() self.consume_task = gevent.spawn(self.consume_task, latch) latch.wait()
def init_env(logger_name): logger = logging.getLogger(logger_name) logger.debug("=================== Enter Debug Level.=====================") # 配置redis RedisCtx.get_instance().host = settings.Redis_Host RedisCtx.get_instance().port = settings.Redis_Port # 初始化 metrics 服务 try: from threathunter_common.metrics.metricsagent import MetricsAgent except ImportError: logger.error( u"from threathunter_common.metrics.metricsagent import MetricsAgent 失败" ) sys.exit(-1) MetricsAgent.get_instance().initialize_by_dict(settings.metrics_dict) logger.info(u"成功初始化metrics服务: {}.".format(MetricsAgent.get_instance().m)) return logger
def write_click_counts(counter, date): """ 向redis记录访问次数记录 counter: {key: count} date: 使用的持久化数据的文件夹的时间,精确到小时 """ r = RedisCtx.get_instance().redis with r.pipeline() as pipe: pipe.multi() for k, v in counter.iteritems(): pipe.hset(k, date, v) pipe.execute()
def consume_task(self): while self.running: try: msg = RedisCtx.get_instance().redis.blpop([self.name], timeout=1) if not msg: continue self.cache.put(msg[1], timeout=10) except Queue.Full: self.full_errors += 1 except Exception as err: print err time.sleep(1)
def consume_task(self, latch): while self.running: try: r = RedisCtx.get_instance().redis self.pubsub = r.pubsub() self.pubsub.subscribe(self.name) for item in self.pubsub.listen(): if item["type"] == "subscribe": latch.countdown() elif item["type"] == "message": try: self.cache.put(item["data"], timeout=10) except Queue.Full: self.full_errors += 1 except Exception as err: print err
def consume_task(self): while self.running: try: msg = RedisCtx.get_instance().redis.blpop([self.name], timeout=1) if not msg: continue self.cache.put(msg[1], timeout=10) except gevent.queue.Full: self.full_errors += 1 except Exception: import traceback logging.error(traceback.format_exc()) finally: gevent.sleep(0.5)
def qsize(self): return RedisCtx.get_instance().redis.llen(self.name)
def put_nowait(self, obj, routing_key, durable=False): idx = hash(routing_key) % self.count name = self.names[idx] RedisCtx.get_instance().redis.rpush(name, str(obj))
def put_nowait(self, obj, routing_key, durable=False): RedisCtx.get_instance().redis.publish(self.name, str(obj))
def setup_method(self, method): RedisCtx.get_instance().host = "127.0.0.1" RedisCtx.get_instance().port = 6379
# -*- coding: utf-8 -*- import time from threathunter_common.redis.redisctx import RedisCtx from threathunter_common.metrics.metricsagent import MetricsAgent nodes = [ dict(host="127.0.0.1", port=7000), dict(host="127.0.0.1", port=7001), dict(host="127.0.0.1", port=7002), dict(host="127.0.0.1", port=7003), dict(host="127.0.0.1", port=7004), dict(host="127.0.0.1", port=7005), ] RedisCtx.get_instance().nodes = nodes metrics_dict = { "app": "nebula_web", "redis": { "type": "redis", "host": "127.0.0.1", "port": 7000, "password": "******", "nodes": nodes }, "influxdb": { "type": "influxdb", "url": "127.0.0.1", "username": "******", "password": "******" }, "server": "redis"
def compute_statistic(specify_db_path=None): """ """ # 初始化redis实例 RedisCtx.get_instance().host = settings.Redis_Host RedisCtx.get_instance().port = settings.Redis_Port MetricsAgent.get_instance().initialize_by_dict(metrics_dict) # 获取日志文件们所在的路径 db_path, logs_path = Index.get_log_paths(specify_db_path) work_hour = db_path.rsplit('/', 1)[-1] t = datetime.strptime(work_hour, settings.LogPath_Format) settings.Working_TS = time.mktime( (t.year, t.month, t.day, t.hour, t.minute, t.second, 0, 0, 0)) settings.Working_DAY = int( time.mktime((t.year, t.month, t.day, 0, 0, 0, 0, 0, 0))) logger.debug(DEBUG_PREFIX + 'working_hour:%s working_ts:%s, len:%s', work_hour, settings.Working_TS, len(str(settings.Working_TS))) # 从每个小时日志文件夹中获取 record schema && record header utils.load_record_settings(db_path) # 重新生成索引python_index目录 Index.regen_index(db_path) # 获取所有策略权重 utils.get_strategies_weigh() # 获取compute variables compute_variables = utils.fetch_compute_variables() logger.debug(DEBUG_PREFIX + '获得的计算变量们是:%s', [_['name'] for _ in compute_variables if _['name']]) # 新增变量时的调试本地的变量文件, 文件里面就可以只有单独的变量树来避免等很久 # import json # with open('/home/threathunter/nebula/nebula_web/venv/lib/python2.7/site-packages/nebula_utils/unittests/offline.json', 'r') as f: # compute_variables = json.load(f) # cvs = [ ComputeVariableHandler.get_compute_variable(**_) for _ in compute_variables] # dag.add_nodes(cvs) # 遍历日志离线统计 compute_dag(compute_variables, logs_path) # 注册生成风险事件的回调函数 Hook_Functions.append(gen_risk_incidents) # 统计点击量的counter Hook_Functions.append(gen_click_counter) # Hook_Functions.append(gen_related_counter) # 注册统计user profile的回调函数 Hook_Functions.append(gen_visit_profile) # 统计notices过去一个小时的数据 ioloop.IOLoop.current().run_sync(gen_notice_statistics) # 聚合notices过去一个小时的metrics last = millis_now() logger.info("开始merge history metrics") merge_history_metrics('default', 'web.notices', 'sum', group_tags=['test', 'strategy', 'location', 'url']) now = millis_now() logger.info("时间消耗:{}ms".format(now - last)) last = now # 清理统计数据目录 logger.info("开始清理统计数据目录") stat_db_tmp_path, stat_db_path = get_stat_db_path(db_path) now = millis_now() logger.info("时间消耗:{}ms".format(now - last)) last = now # 持久化统计数据 logger.info("开始持久化统计数据") write_statistic_data(stat_db_tmp_path) now = millis_now() logger.info("时间消耗:{}ms".format(now - last)) last = now # 统计完成, 临时统计目录改名为正式, 提供查询服务 logger.info("开始移动目录") shutil.move(stat_db_tmp_path, stat_db_path) now = millis_now() logger.info("时间消耗:{}ms".format(now - last)) last = now # 定时脚本统计完成后,调用web API清除缓存,刷新数据 utils.clean_cache() now = millis_now() logger.info("时间消耗:{}ms".format(now - last))
def setup_class(self): RedisCtx.get_instance().host = "127.0.0.1" RedisCtx.get_instance().port = 6379 print "start to init service" TestService.init_services() print "successfully started the services"
def webui(ctx, port, debug): if debug: logger.setLevel(logging.DEBUG) from nebula.views import ( auth, general, influxdbproxy, config, metrics, notice, system_perf, system_log, nebula_config, strategy, checkrisk, strategy_default, config_default, read_batch, alarm, network, user, group, risk_incident, logparser, permission, notice_export, notice_report, strategy_export, upgrade, logquery, follow, event_model, event_model_default, variable_model, variable_model_default) # 注册nebula_backend type2class from nebula_meta.model import variable_meta print variable_meta.VariableMeta.TYPE2Class # 初始化 redis 服务 try: from threathunter_common.redis.redisctx import RedisCtx except ImportError: logger.error( u"from threathunter_common.redis.redisctx import RedisCtx 失败.") sys.exit(-1) RedisCtx.get_instance().host = settings.Redis_Host RedisCtx.get_instance().port = settings.Redis_Port logger.debug(u"成功初始化redis: {}".format(RedisCtx.get_instance())) # 初始化 metrics 服务 try: from threathunter_common.metrics.metricsagent import MetricsAgent except ImportError: logger.error( u"from threathunter_common.metrics.metricsagent import MetricsAgent 失败" ) sys.exit(-1) MetricsAgent.get_instance().initialize_by_dict(metrics_dict) logger.debug(u"成功初始化metrics服务: {}.".format(MetricsAgent.get_instance().m)) # 初始化 babel service # from nebula.services import babel # babel.set_mode(settings.Babel_Mode) # logger.debug(u"成功初始化 {} 模式的babel服务.".format(babel.mode)) # 启动 performance metrics logger from nebula.services.metricspoller import HistoryMetricsPoller his_metrics = HistoryMetricsPoller() his_metrics.start() logger.debug(u"成功启动metrics性能日志服务.") # # 启动 定期清理notice任务 # from nebula.services.notice_cleaner import NoticeCleaner # cleaner = NoticeCleaner() # cleaner.start() # logger.debug(u"成功启动定期清理notice服务.") # 初始化 dbcontext from nebula.dao.DBDataCache import dbcontext dbcontext.init() # # 启动 NoticeRPCServer # template_path = settings.Notice_RPC_Template_Path#os.path.join(os.path.dirname(__file__), "templates") # from nebula.services.notice_server import NoticeRPCServer # notice_server = NoticeRPCServer(template_path) # notice_server.start() # load取cache们 from nebula.dao.cache import Cache_Init_Functions, init_caches # 策略权重的缓存 from nebula.dao.strategy_dao import init_strategy_weigh Cache_Init_Functions.append(init_strategy_weigh) init_caches() # load session auth code init_session_jar() # 启动 ESLogSendServer # from nebula.services.eslog_sender import ESLogSenderServer # eslogserver = ESLogSenderServer() # eslogserver.start() urls = [ # 通用模块 (r"/", general.IndexHandler), (r"/project", general.ProjectHandler), (r"/user", general.LoginHandler), (r"/nebula/web/config", general.WebConfigHandler), (r"/auth/register", auth.RegisterHandler), # 权限模块 (r"/auth/login", auth.LoginHandler), (r"/auth/logout", auth.LogoutHandler), (r"/auth/changepwd", auth.ChangePasswordHandler), (r"/auth/users", user.UserListHandler), (r"/auth/users/(.*)", user.UserQueryHandler), (r"/auth/groups", group.GroupListHandler), (r"/auth/groups/(.*)", group.GroupQueryHandler), (r"/auth/permissions", permission.PermissionListHandler), (r"/auth/permissions/(.*)/(.*)", permission.PermissionQueryHandler), (r"/auth/strategy_access", group.StrategyAccessHandler), (r"/auth/privileges", group.PrivilegesHandler), # 系统模块 (r"/system/performance/digest", system_perf.SystemPerformanceHandler), (r'/system/log', system_log.LogInfoHandler), (r'/geo/all_city', general.AllCityHandler), (r'/geo/all_province', general.AllProvinceHandler), (r'/platform/geoinfo', general.GeoStatsHandler), # 默认的策略、配置、变量们 (r"/default/variable_models/variable/(.*)/(.*)", variable_model_default.VariableModelQueryHandler), (r"/default/variable_models", variable_model_default.VariableModelListHandler), (r"/default/strategies", strategy_default.StrategyListHandler), (r"/default/strategies/strategy/(.*)/(.*)", strategy_default.StrategyQueryHandler), (r"/default/strategies/changestatus/", strategy_default.StrategyStatusHandler), (r"/default/config", config_default.ConfigListHandler), (r"/default/configproperties", config_default.ConfigPropertiesHandler), (r"/default/config/(.*)", config_default.ConfigHandler), (r"/default/event_models/event/(.*)/(.*)", event_model_default.EventQueryHandler), (r"/default/event_models", event_model_default.EventModelListHandler), # 定制配置接口 (r"/platform/config", config.ConfigListHandler), (r"/platform/configproperties", config.ConfigPropertiesHandler), (r"/platform/config/(.*)", config.ConfigHandler), # 定制事件接口 (r"/platform/event_models/event/(.*)/(.*)", event_model.EventQueryHandler), (r"/platform/event_models", event_model.EventListHandler), (r"/platform/event_models_beautify", event_model.EventModelBeautifyHandler), # 定制变量接口 (r"/platform/variable_models/variable/(.*)/(.*)", variable_model.VariableModelQueryHandler), (r"/platform/variable_models", variable_model.VariableModelListHandler), (r"/platform/variable_models_beautify", variable_model.VariableModelBeautifyHandler), # 定制策略接口 (r"/nebula/strategies", strategy.StrategyListHandler), (r"/nebula/strategies/strategy/(.*)/(.*)", strategy.StrategyQueryHandler), (r"/nebula/strategies/changestatus/", strategy.StrategyStatusHandler), (r"/nebula/strategies/delete", strategy.StrategyBatchDelHandler), (r"/nebula/strategy/import", strategy_export.StrategyImportHandler), (r"/nebula/strategy/export", strategy_export.StrategyExportHandler), (r"/nebula/strategy/export/(.*)", strategy_export.StrategyDownloadHandler), (r"/nebula/strategyweigh", strategy.StrategyWeighHandler), (r"/nebula/tags", strategy.TagsHandler), (r"/nebula/tag/(.*)", strategy.TagQueryHandler), (r"/nebula/glossary", nebula_config.VariableGlossaryHandler), (r"/nebula/events", nebula_config.NebulaUIEventsHandler), (r"/nebula/variables", nebula_config.NebulaUIVariablesHandler), (r"/nebula/online/events", nebula_config.NebulaOnlineEventsHandler), (r"/nebula/online/variables", nebula_config.NebulaOnlineVariablesHandler), # 策略定制脚本相关接口 (r"/nebula/NebulaStrategy", nebula_strategy.NebulaStrategy), (r"/nebula/supervisor", Supervisor), # (r"/platform/variabledata/latest/(.*)", variable_value.VariableValueQueryHandler), # (r"/platform/variabledata/top/(.*)", variable_value.VariableValueTopHandler), # (r"/platform/variabledata/list/(.*)", variable_value.VariableValueListHandler), # (r"/platform/variabledata/keytop/(.*)", variable_value.VariableValueKeyTopHandler), # 风险事件相关 (r"/platform/risk_incidents", risk_incident.IncidentListHandler), # (r"/platform/risks/statistics", risk_incident.RisksStatisticsHandler), # (r"/platform/risks/realtime", risk_incident.RisksRealtimeHandler), (r"/platform/risks/history", risk_incident.RisksHistoryHandler), (r'/platform/risks/black_check', risk_incident.BlackHandler), (r"/platform/risks/(.*)", risk_incident.IncidentQueryHandler), (r"/platform/notices/export", notice_export.NoticeExportHandler), (r"/platform/notices/export/(.*)", tornado.web.StaticFileHandler, { "path": settings.NoticeExport_Path }), (r'/platform/stats/notice_report', notice_report.NoticeReportHandler), # 统计数据源通用查询接口 # (r'/platform/stats/online', data_bus.OnlineDataHandler), # (r'/platform/stats/slot', data_bus.SlotDataHandler), # (r'/platform/stats/slot_baseline', data_bus.SlotBaseLineDataHandler), # (r'/platform/stats/offline', data_bus.OfflineDataHandler), # (r'/platform/stats/offline_baseline', data_bus.OfflineBaseLineDataHandler), # (r'/platform/stats/profile', data_bus.ProfileDataHandler), # (r'/platform/stats/offline_serial', data_bus.OfflineSerialDataHandler), # (r'/platform/stats/metrics', data_bus.MetricsDataHandler), # (r'/platform/stats/notice', data_bus.NoticeDataHandler), # (r'/platform/stats/risk_incident', data_bus.RiskIncidentDataHandler), # (r'/platform/stats/geo', data_bus.GEODataHandler), # (r'/platform/stats/threat_map', data_bus.ThreatMapDataHandler), # (r'/platform/stats/clean_cache', data_bus.CleanCacheHandler), # 持久化事件查询数据 (r'/platform/alarm', alarm.AlarmListHandler), (r'/platform/alarm/valid_count', alarm.ValidCountHandler), (r'/platform/alarm/statistics', alarm.StatisticsHandler), (r'/platform/network/statistics', network.NetworkStatisticsHandler), # (r'/platform/alarm/statistics_detail', alarm.StatisticsDetailHandler), # (r'/platform/behavior/strategy_statistic', incident_stat.StrategyStatHandler), (r'/platform/behavior/tag_statistics', alarm.TagStatHandler), # (r'/platform/behavior/start_time',incident_stat.PersistBeginTimeHandler), # (r'/platform/behavior/statistics', incident_stat.IncidentStatsHandler), # (r'/platform/behavior/clicks_detail', incident_stat.ClickDetailHandler), # (r'/platform/behavior/related_statistics', incident_stat.RelatedStatisticHandler), # (r'/platform/behavior/continuous_related_statistic', incident_stat.ContinuousRelatedStatHandler), # (r'/platform/behavior/continuous_top_related_statistic', incident_stat.ContinuousTopRelatedStatHandler), # (r'/platform/behavior/scene_statistic', incident_stat.SceneStatHandler), # (r"/platform/behavior/user_statistics", incident_stat.UserStatHandler), # (r'/platform/behavior/page_statistics', incident_stat.RelatedPageStatisticHandler), # (r'/platform/behavior/top/clicks_location', incident_stat.ClickLocation), # (r'/platform/online/visit_stream', incident_stat.OnlineVisitStreamHandler), # (r'/platform/behavior/visit_stream', incident_stat.OnlineVisitStreamHandler), # (r'/platform/online/clicks_period', incident_stat.OnlineClicksPeriodHandler), # (r'/platform/behavior/clicks_period', incident_stat.OnlineClicksPeriodHandler), # (r'/platform/online/clicks', incident_stat.OnlineClickListHandler), # (r'/platform/behavior/clicks', incident_stat.OnlineClickListHandler), # 日志解析 (r"/platform/logparser", logparser.LogParserListHandler), # 日志查询 # (r'/platform/logquery', incident_stat.LogQuery), # (r'/platform/logquery/(.*)', tornado.web.StaticFileHandler, {'path':settings.LogQuery_Path}), # (r"/platform/logquery_config", logquery.LogQueryConfigHandler), # metrics (r"/platform/metrics/(.*)", metrics.MetricsHandler), (r"/platform/batchmetrics/", metrics.BatchMetricsHandler), (r"/metricsproxy/.*", influxdbproxy.InfluxdbProxyHandler), # 黑名单相关 (r"/platform/notices", notice.NoticeListHandler), (r"/platform/notices/trigger_event", notice.TriggerEventHandler), (r"/platform/noticestats", notice.NoticeStatsHandler), (r"/platform/bwlist", notice.NoticeBWListHandler), (r"/platform/noticetimestamps", notice.NoticeTimestampListHandler), (r"/checkRisk", checkrisk.CheckRiskHandler), (r"/checkRiskTest", checkrisk.CheckRiskHandler), (r"/checkRiskTest/GiveMeAccept", checkrisk.GiveMeAcceptHandler), (r"/checkRiskTest/GiveMeReview", checkrisk.GiveMeReviewHandler), (r"/checkRiskTest/GiveMeReject", checkrisk.GiveMeRejectHandler), (r"/checkRiskTest/GiveMeNothing", checkrisk.GiveMeNothingHandler), (r"/checkRiskTest/GiveMeError", checkrisk.GiveMeErrorHandler), # (r"/platform/monitor/riskyitems", monitor.RiskyItemsHandler), # (r"/platform/monitor/toptargets", monitor.TopTargetsHandler), # (r"/platform/monitor/topcities", monitor.TopCitiesHandler), (r"/platform/data/export/notice", notice.NoticeExportHandler), (r"/platform/batchCheckRisk/bwlist", read_batch.BatchBWListHandler), # 档案查询 # (r"/platform/stats/profile", profile.ProfileHandler), # (r"/platform/stats/page_risk", profile.ProfilePageRiskHandler), # (r"/platform/stats/account_risk", profile.ProfileAccountRiskHandler), # 爬虫统计 (r"/platform/follow_keyword", follow.FollowKeywordHandler), (r"/platform/follow_keyword/(.*)", follow.FollowKeywordAnotherHandler), # 更新nebula各组件 (r"/platform/api/upgrade", upgrade.UpgradeHandler), # restful相关接口 (r"/restful/", general.SwaggerUIHandler, { 'assets_path': settings.Swagger_Assets_Path }), (r"/restful/(.*)", tornado.web.StaticFileHandler, { 'path': settings.Swagger_Assets_Path }), ] # 注册 grafana url metrics = [ (r'/metrics/(.*)', general.CustomizedFileHandler, { 'path': settings.GRAFANA_PATH }), ] urls.extend(metrics) settings.Tornado_Setting["compress_response"] = True settings.Tornado_Setting["job_worker"] = 10 utils.executor = ThreadPoolExecutor( max_workers=settings.Tornado_Setting.get('job_worker', 10)) application = tornado.web.Application(urls, **settings.Tornado_Setting) # 注册 restful api url application_api_wrapper(application, authenticated, need_auth=False) #restful api not need auth # 注册nebula_strategy from nebula.views.nebula_config import context as nebula_context def load_event_schema(): events = [_.get_dict() for _ in nebula_context.nebula_events] event_schema = dict() for e in events: properties = {p["name"]: p["type"] for p in e["properties"]} event_schema[e["name"]] = properties return event_schema def load_variable_schema(): variables = [_.get_dict() for _ in nebula_context.nebula_variables] variable_schema = dict() for v in variables: variable_schema[v["name"]] = {} return variable_schema # """定时任务处理,此后需要定时任务都可往里面加,无需加机器上的crontab""" # scheduler = BackgroundScheduler() # # 添加调度任务 # # 触发器选择 interval(间隔性),间隔时长为 60 秒 # scheduler.add_job(scheduler_tasks[0], 'interval', seconds=60) # # 启动调度任务 # scheduler.start() # 定时60秒启动一次 tornado.ioloop.PeriodicCallback(scheduler_tasks[0], callback_time=1000 * 60).start() tornado.options.parse_command_line() http_server = tornado.httpserver.HTTPServer(application, xheaders=True) http_server.listen(str(port), settings.WebUI_Address) click.echo("Nebula Web Start, Port:%s" % port) tornado.ioloop.IOLoop.instance().start()