def put_nowait(self, obj, routing_key, durable=False):
if self.lazy_limit and self.qsize_diff < self.qsize_diff_limit:
pass
elif self.full():
raise Queue.Full
else:
self.qsize_diff = 0
with self.lock:
self.qsize_diff += 1
RedisCtx.get_instance().redis.rpush(self.name, str(obj))
def init_redis():
from complexconfig.configcontainer import configcontainer
host = configcontainer.get_config("sniffer").get_string("sniffer.redis.host")
port = configcontainer.get_config("sniffer").get_int("sniffer.redis.port")
password = configcontainer.get_config("sniffer").get_string("sniffer.redis.password", "")
from threathunter_common.redis.redisctx import RedisCtx
RedisCtx.get_instance().host = host
RedisCtx.get_instance().port = port
RedisCtx.get_instance().password = password
print_with_time("successfully init redis[host={},port={},password={}]".format(host, port, "*"*len(password)))
def get_user_from_session(session):
if not session:
return ""
# 1. try load from local cache
result = local_cache.get(session)
if result == NOT_EXIST_USER:
# should searched redis in the past and no value is found
return ""
elif result:
# there is value
return result
else:
# do redis ops
pass
# 2. try load from redis cache
redis_key = "user_for_session_{}".format(session)
result = RedisCtx.get_instance().redis.get(redis_key)
# populate the cache
if result:
local_cache[session] = result
else:
local_cache[session] = NOT_EXIST_USER
return result
def start_consuming(self):
self.running = True
r = RedisCtx.get_instance().redis
self.pubsub = r.pubsub()
self.pubsub.subscribe(self.name)
latch = CountDownLatch()
self.consume_task = gevent.spawn(self.consume_task, latch)
latch.wait()
def init_env(logger_name):
logger = logging.getLogger(logger_name)
logger.debug("=================== Enter Debug Level.=====================")
# 配置redis
RedisCtx.get_instance().host = settings.Redis_Host
RedisCtx.get_instance().port = settings.Redis_Port
# 初始化 metrics 服务
try:
from threathunter_common.metrics.metricsagent import MetricsAgent
except ImportError:
logger.error(
u"from threathunter_common.metrics.metricsagent import MetricsAgent 失败"
)
sys.exit(-1)
MetricsAgent.get_instance().initialize_by_dict(settings.metrics_dict)
logger.info(u"成功初始化metrics服务: {}.".format(MetricsAgent.get_instance().m))
return logger
def write_click_counts(counter, date):
"""
向redis记录访问次数记录
counter: {key: count}
date: 使用的持久化数据的文件夹的时间，精确到小时
"""
r = RedisCtx.get_instance().redis
with r.pipeline() as pipe:
pipe.multi()
for k, v in counter.iteritems():
pipe.hset(k, date, v)
pipe.execute()
def consume_task(self):
while self.running:
try:
msg = RedisCtx.get_instance().redis.blpop([self.name],
timeout=1)
if not msg:
continue
self.cache.put(msg[1], timeout=10)
except Queue.Full:
self.full_errors += 1
except Exception as err:
print err
time.sleep(1)
def consume_task(self, latch):
while self.running:
try:
r = RedisCtx.get_instance().redis
self.pubsub = r.pubsub()
self.pubsub.subscribe(self.name)
for item in self.pubsub.listen():
if item["type"] == "subscribe":
latch.countdown()
elif item["type"] == "message":
try:
self.cache.put(item["data"], timeout=10)
except Queue.Full:
self.full_errors += 1
except Exception as err:
print err
def consume_task(self):
while self.running:
try:
msg = RedisCtx.get_instance().redis.blpop([self.name], timeout=1)
if not msg:
continue
self.cache.put(msg[1], timeout=10)
except gevent.queue.Full:
self.full_errors += 1
except Exception:
import traceback
logging.error(traceback.format_exc())
finally:
gevent.sleep(0.5)
def qsize(self):
return RedisCtx.get_instance().redis.llen(self.name)
def put_nowait(self, obj, routing_key, durable=False):
idx = hash(routing_key) % self.count
name = self.names[idx]
RedisCtx.get_instance().redis.rpush(name, str(obj))
def put_nowait(self, obj, routing_key, durable=False):
RedisCtx.get_instance().redis.publish(self.name, str(obj))
def setup_method(self, method):
RedisCtx.get_instance().host = "127.0.0.1"
RedisCtx.get_instance().port = 6379
# -*- coding: utf-8 -*-
import time
from threathunter_common.redis.redisctx import RedisCtx
from threathunter_common.metrics.metricsagent import MetricsAgent
nodes = [
dict(host="127.0.0.1", port=7000),
dict(host="127.0.0.1", port=7001),
dict(host="127.0.0.1", port=7002),
dict(host="127.0.0.1", port=7003),
dict(host="127.0.0.1", port=7004),
dict(host="127.0.0.1", port=7005),
]
RedisCtx.get_instance().nodes = nodes
metrics_dict = {
"app": "nebula_web",
"redis": {
"type": "redis",
"host": "127.0.0.1",
"port": 7000,
"password": "******",
"nodes": nodes
},
"influxdb": {
"type": "influxdb",
"url": "127.0.0.1",
"username": "******",
"password": "******"
},
"server": "redis"
def compute_statistic(specify_db_path=None):
"""
"""
# 初始化redis实例
RedisCtx.get_instance().host = settings.Redis_Host
RedisCtx.get_instance().port = settings.Redis_Port
MetricsAgent.get_instance().initialize_by_dict(metrics_dict)
# 获取日志文件们所在的路径
db_path, logs_path = Index.get_log_paths(specify_db_path)
work_hour = db_path.rsplit('/', 1)[-1]
t = datetime.strptime(work_hour, settings.LogPath_Format)
settings.Working_TS = time.mktime(
(t.year, t.month, t.day, t.hour, t.minute, t.second, 0, 0, 0))
settings.Working_DAY = int(
time.mktime((t.year, t.month, t.day, 0, 0, 0, 0, 0, 0)))
logger.debug(DEBUG_PREFIX + 'working_hour:%s working_ts:%s, len:%s',
work_hour, settings.Working_TS, len(str(settings.Working_TS)))
# 从每个小时日志文件夹中获取 record schema && record header
utils.load_record_settings(db_path)
# 重新生成索引python_index目录
Index.regen_index(db_path)
# 获取所有策略权重
utils.get_strategies_weigh()
# 获取compute variables
compute_variables = utils.fetch_compute_variables()
logger.debug(DEBUG_PREFIX + '获得的计算变量们是:%s',
[_['name'] for _ in compute_variables if _['name']])
# 新增变量时的调试本地的变量文件, 文件里面就可以只有单独的变量树来避免等很久
# import json
# with open('/home/threathunter/nebula/nebula_web/venv/lib/python2.7/site-packages/nebula_utils/unittests/offline.json', 'r') as f:
# compute_variables = json.load(f)
# cvs = [ ComputeVariableHandler.get_compute_variable(**_) for _ in compute_variables]
# dag.add_nodes(cvs)
# 遍历日志离线统计
compute_dag(compute_variables, logs_path)
# 注册生成风险事件的回调函数
Hook_Functions.append(gen_risk_incidents)
# 统计点击量的counter
Hook_Functions.append(gen_click_counter)
# Hook_Functions.append(gen_related_counter)
# 注册统计user profile的回调函数
Hook_Functions.append(gen_visit_profile)
# 统计notices过去一个小时的数据
ioloop.IOLoop.current().run_sync(gen_notice_statistics)
# 聚合notices过去一个小时的metrics
last = millis_now()
logger.info("开始merge history metrics")
merge_history_metrics('default',
'web.notices',
'sum',
group_tags=['test', 'strategy', 'location', 'url'])
now = millis_now()
logger.info("时间消耗：{}ms".format(now - last))
last = now
# 清理统计数据目录
logger.info("开始清理统计数据目录")
stat_db_tmp_path, stat_db_path = get_stat_db_path(db_path)
now = millis_now()
logger.info("时间消耗：{}ms".format(now - last))
last = now
# 持久化统计数据
logger.info("开始持久化统计数据")
write_statistic_data(stat_db_tmp_path)
now = millis_now()
logger.info("时间消耗：{}ms".format(now - last))
last = now
# 统计完成, 临时统计目录改名为正式, 提供查询服务
logger.info("开始移动目录")
shutil.move(stat_db_tmp_path, stat_db_path)
now = millis_now()
logger.info("时间消耗：{}ms".format(now - last))
last = now
# 定时脚本统计完成后，调用web API清除缓存，刷新数据
utils.clean_cache()
now = millis_now()
logger.info("时间消耗：{}ms".format(now - last))
def setup_class(self):
RedisCtx.get_instance().host = "127.0.0.1"
RedisCtx.get_instance().port = 6379
print "start to init service"
TestService.init_services()
print "successfully started the services"
def webui(ctx, port, debug):
if debug:
logger.setLevel(logging.DEBUG)
from nebula.views import (
auth, general, influxdbproxy, config, metrics, notice, system_perf,
system_log, nebula_config, strategy, checkrisk, strategy_default,
config_default, read_batch, alarm, network, user, group, risk_incident,
logparser, permission, notice_export, notice_report, strategy_export,
upgrade, logquery, follow, event_model, event_model_default,
variable_model, variable_model_default)
# 注册nebula_backend type2class
from nebula_meta.model import variable_meta
print variable_meta.VariableMeta.TYPE2Class
# 初始化 redis 服务
try:
from threathunter_common.redis.redisctx import RedisCtx
except ImportError:
logger.error(
u"from threathunter_common.redis.redisctx import RedisCtx 失败.")
sys.exit(-1)
RedisCtx.get_instance().host = settings.Redis_Host
RedisCtx.get_instance().port = settings.Redis_Port
logger.debug(u"成功初始化redis: {}".format(RedisCtx.get_instance()))
# 初始化 metrics 服务
try:
from threathunter_common.metrics.metricsagent import MetricsAgent
except ImportError:
logger.error(
u"from threathunter_common.metrics.metricsagent import MetricsAgent 失败"
)
sys.exit(-1)
MetricsAgent.get_instance().initialize_by_dict(metrics_dict)
logger.debug(u"成功初始化metrics服务: {}.".format(MetricsAgent.get_instance().m))
# 初始化 babel service
# from nebula.services import babel
# babel.set_mode(settings.Babel_Mode)
# logger.debug(u"成功初始化 {} 模式的babel服务.".format(babel.mode))
# 启动 performance metrics logger
from nebula.services.metricspoller import HistoryMetricsPoller
his_metrics = HistoryMetricsPoller()
his_metrics.start()
logger.debug(u"成功启动metrics性能日志服务.")
# # 启动 定期清理notice任务
# from nebula.services.notice_cleaner import NoticeCleaner
# cleaner = NoticeCleaner()
# cleaner.start()
# logger.debug(u"成功启动定期清理notice服务.")
# 初始化 dbcontext
from nebula.dao.DBDataCache import dbcontext
dbcontext.init()
# # 启动 NoticeRPCServer
# template_path = settings.Notice_RPC_Template_Path#os.path.join(os.path.dirname(__file__), "templates")
# from nebula.services.notice_server import NoticeRPCServer
# notice_server = NoticeRPCServer(template_path)
# notice_server.start()
# load取cache们
from nebula.dao.cache import Cache_Init_Functions, init_caches
# 策略权重的缓存
from nebula.dao.strategy_dao import init_strategy_weigh
Cache_Init_Functions.append(init_strategy_weigh)
init_caches()
# load session auth code
init_session_jar()
# 启动 ESLogSendServer
# from nebula.services.eslog_sender import ESLogSenderServer
# eslogserver = ESLogSenderServer()
# eslogserver.start()
urls = [
# 通用模块
(r"/", general.IndexHandler),
(r"/project", general.ProjectHandler),
(r"/user", general.LoginHandler),
(r"/nebula/web/config", general.WebConfigHandler),
(r"/auth/register", auth.RegisterHandler),
# 权限模块
(r"/auth/login", auth.LoginHandler),
(r"/auth/logout", auth.LogoutHandler),
(r"/auth/changepwd", auth.ChangePasswordHandler),
(r"/auth/users", user.UserListHandler),
(r"/auth/users/(.*)", user.UserQueryHandler),
(r"/auth/groups", group.GroupListHandler),
(r"/auth/groups/(.*)", group.GroupQueryHandler),
(r"/auth/permissions", permission.PermissionListHandler),
(r"/auth/permissions/(.*)/(.*)", permission.PermissionQueryHandler),
(r"/auth/strategy_access", group.StrategyAccessHandler),
(r"/auth/privileges", group.PrivilegesHandler),
# 系统模块
(r"/system/performance/digest", system_perf.SystemPerformanceHandler),
(r'/system/log', system_log.LogInfoHandler),
(r'/geo/all_city', general.AllCityHandler),
(r'/geo/all_province', general.AllProvinceHandler),
(r'/platform/geoinfo', general.GeoStatsHandler),
# 默认的策略、配置、变量们
(r"/default/variable_models/variable/(.*)/(.*)",
variable_model_default.VariableModelQueryHandler),
(r"/default/variable_models",
variable_model_default.VariableModelListHandler),
(r"/default/strategies", strategy_default.StrategyListHandler),
(r"/default/strategies/strategy/(.*)/(.*)",
strategy_default.StrategyQueryHandler),
(r"/default/strategies/changestatus/",
strategy_default.StrategyStatusHandler),
(r"/default/config", config_default.ConfigListHandler),
(r"/default/configproperties", config_default.ConfigPropertiesHandler),
(r"/default/config/(.*)", config_default.ConfigHandler),
(r"/default/event_models/event/(.*)/(.*)",
event_model_default.EventQueryHandler),
(r"/default/event_models", event_model_default.EventModelListHandler),
# 定制配置接口
(r"/platform/config", config.ConfigListHandler),
(r"/platform/configproperties", config.ConfigPropertiesHandler),
(r"/platform/config/(.*)", config.ConfigHandler),
# 定制事件接口
(r"/platform/event_models/event/(.*)/(.*)",
event_model.EventQueryHandler),
(r"/platform/event_models", event_model.EventListHandler),
(r"/platform/event_models_beautify",
event_model.EventModelBeautifyHandler),
# 定制变量接口
(r"/platform/variable_models/variable/(.*)/(.*)",
variable_model.VariableModelQueryHandler),
(r"/platform/variable_models",
variable_model.VariableModelListHandler),
(r"/platform/variable_models_beautify",
variable_model.VariableModelBeautifyHandler),
# 定制策略接口
(r"/nebula/strategies", strategy.StrategyListHandler),
(r"/nebula/strategies/strategy/(.*)/(.*)",
strategy.StrategyQueryHandler),
(r"/nebula/strategies/changestatus/", strategy.StrategyStatusHandler),
(r"/nebula/strategies/delete", strategy.StrategyBatchDelHandler),
(r"/nebula/strategy/import", strategy_export.StrategyImportHandler),
(r"/nebula/strategy/export", strategy_export.StrategyExportHandler),
(r"/nebula/strategy/export/(.*)",
strategy_export.StrategyDownloadHandler),
(r"/nebula/strategyweigh", strategy.StrategyWeighHandler),
(r"/nebula/tags", strategy.TagsHandler),
(r"/nebula/tag/(.*)", strategy.TagQueryHandler),
(r"/nebula/glossary", nebula_config.VariableGlossaryHandler),
(r"/nebula/events", nebula_config.NebulaUIEventsHandler),
(r"/nebula/variables", nebula_config.NebulaUIVariablesHandler),
(r"/nebula/online/events", nebula_config.NebulaOnlineEventsHandler),
(r"/nebula/online/variables",
nebula_config.NebulaOnlineVariablesHandler),
# 策略定制脚本相关接口
(r"/nebula/NebulaStrategy", nebula_strategy.NebulaStrategy),
(r"/nebula/supervisor", Supervisor),
# (r"/platform/variabledata/latest/(.*)", variable_value.VariableValueQueryHandler),
# (r"/platform/variabledata/top/(.*)", variable_value.VariableValueTopHandler),
# (r"/platform/variabledata/list/(.*)", variable_value.VariableValueListHandler),
# (r"/platform/variabledata/keytop/(.*)", variable_value.VariableValueKeyTopHandler),
# 风险事件相关
(r"/platform/risk_incidents", risk_incident.IncidentListHandler),
# (r"/platform/risks/statistics", risk_incident.RisksStatisticsHandler),
# (r"/platform/risks/realtime", risk_incident.RisksRealtimeHandler),
(r"/platform/risks/history", risk_incident.RisksHistoryHandler),
(r'/platform/risks/black_check', risk_incident.BlackHandler),
(r"/platform/risks/(.*)", risk_incident.IncidentQueryHandler),
(r"/platform/notices/export", notice_export.NoticeExportHandler),
(r"/platform/notices/export/(.*)", tornado.web.StaticFileHandler, {
"path": settings.NoticeExport_Path
}),
(r'/platform/stats/notice_report', notice_report.NoticeReportHandler),
# 统计数据源通用查询接口
# (r'/platform/stats/online', data_bus.OnlineDataHandler),
# (r'/platform/stats/slot', data_bus.SlotDataHandler),
# (r'/platform/stats/slot_baseline', data_bus.SlotBaseLineDataHandler),
# (r'/platform/stats/offline', data_bus.OfflineDataHandler),
# (r'/platform/stats/offline_baseline', data_bus.OfflineBaseLineDataHandler),
# (r'/platform/stats/profile', data_bus.ProfileDataHandler),
# (r'/platform/stats/offline_serial', data_bus.OfflineSerialDataHandler),
# (r'/platform/stats/metrics', data_bus.MetricsDataHandler),
# (r'/platform/stats/notice', data_bus.NoticeDataHandler),
# (r'/platform/stats/risk_incident', data_bus.RiskIncidentDataHandler),
# (r'/platform/stats/geo', data_bus.GEODataHandler),
# (r'/platform/stats/threat_map', data_bus.ThreatMapDataHandler),
# (r'/platform/stats/clean_cache', data_bus.CleanCacheHandler),
# 持久化事件查询数据
(r'/platform/alarm', alarm.AlarmListHandler),
(r'/platform/alarm/valid_count', alarm.ValidCountHandler),
(r'/platform/alarm/statistics', alarm.StatisticsHandler),
(r'/platform/network/statistics', network.NetworkStatisticsHandler),
# (r'/platform/alarm/statistics_detail', alarm.StatisticsDetailHandler),
# (r'/platform/behavior/strategy_statistic', incident_stat.StrategyStatHandler),
(r'/platform/behavior/tag_statistics', alarm.TagStatHandler),
# (r'/platform/behavior/start_time',incident_stat.PersistBeginTimeHandler),
# (r'/platform/behavior/statistics', incident_stat.IncidentStatsHandler),
# (r'/platform/behavior/clicks_detail', incident_stat.ClickDetailHandler),
# (r'/platform/behavior/related_statistics', incident_stat.RelatedStatisticHandler),
# (r'/platform/behavior/continuous_related_statistic', incident_stat.ContinuousRelatedStatHandler),
# (r'/platform/behavior/continuous_top_related_statistic', incident_stat.ContinuousTopRelatedStatHandler),
# (r'/platform/behavior/scene_statistic', incident_stat.SceneStatHandler),
# (r"/platform/behavior/user_statistics", incident_stat.UserStatHandler),
# (r'/platform/behavior/page_statistics', incident_stat.RelatedPageStatisticHandler),
# (r'/platform/behavior/top/clicks_location', incident_stat.ClickLocation),
# (r'/platform/online/visit_stream', incident_stat.OnlineVisitStreamHandler),
# (r'/platform/behavior/visit_stream', incident_stat.OnlineVisitStreamHandler),
# (r'/platform/online/clicks_period', incident_stat.OnlineClicksPeriodHandler),
# (r'/platform/behavior/clicks_period', incident_stat.OnlineClicksPeriodHandler),
# (r'/platform/online/clicks', incident_stat.OnlineClickListHandler),
# (r'/platform/behavior/clicks', incident_stat.OnlineClickListHandler),
# 日志解析
(r"/platform/logparser", logparser.LogParserListHandler),
# 日志查询
# (r'/platform/logquery', incident_stat.LogQuery),
# (r'/platform/logquery/(.*)', tornado.web.StaticFileHandler, {'path':settings.LogQuery_Path}),
# (r"/platform/logquery_config", logquery.LogQueryConfigHandler),
# metrics
(r"/platform/metrics/(.*)", metrics.MetricsHandler),
(r"/platform/batchmetrics/", metrics.BatchMetricsHandler),
(r"/metricsproxy/.*", influxdbproxy.InfluxdbProxyHandler),
# 黑名单相关
(r"/platform/notices", notice.NoticeListHandler),
(r"/platform/notices/trigger_event", notice.TriggerEventHandler),
(r"/platform/noticestats", notice.NoticeStatsHandler),
(r"/platform/bwlist", notice.NoticeBWListHandler),
(r"/platform/noticetimestamps", notice.NoticeTimestampListHandler),
(r"/checkRisk", checkrisk.CheckRiskHandler),
(r"/checkRiskTest", checkrisk.CheckRiskHandler),
(r"/checkRiskTest/GiveMeAccept", checkrisk.GiveMeAcceptHandler),
(r"/checkRiskTest/GiveMeReview", checkrisk.GiveMeReviewHandler),
(r"/checkRiskTest/GiveMeReject", checkrisk.GiveMeRejectHandler),
(r"/checkRiskTest/GiveMeNothing", checkrisk.GiveMeNothingHandler),
(r"/checkRiskTest/GiveMeError", checkrisk.GiveMeErrorHandler),
# (r"/platform/monitor/riskyitems", monitor.RiskyItemsHandler),
# (r"/platform/monitor/toptargets", monitor.TopTargetsHandler),
# (r"/platform/monitor/topcities", monitor.TopCitiesHandler),
(r"/platform/data/export/notice", notice.NoticeExportHandler),
(r"/platform/batchCheckRisk/bwlist", read_batch.BatchBWListHandler),
# 档案查询
# (r"/platform/stats/profile", profile.ProfileHandler),
# (r"/platform/stats/page_risk", profile.ProfilePageRiskHandler),
# (r"/platform/stats/account_risk", profile.ProfileAccountRiskHandler),
# 爬虫统计
(r"/platform/follow_keyword", follow.FollowKeywordHandler),
(r"/platform/follow_keyword/(.*)", follow.FollowKeywordAnotherHandler),
# 更新nebula各组件
(r"/platform/api/upgrade", upgrade.UpgradeHandler),
# restful相关接口
(r"/restful/", general.SwaggerUIHandler, {
'assets_path': settings.Swagger_Assets_Path
}),
(r"/restful/(.*)", tornado.web.StaticFileHandler, {
'path': settings.Swagger_Assets_Path
}),
]
# 注册 grafana url
metrics = [
(r'/metrics/(.*)', general.CustomizedFileHandler, {
'path': settings.GRAFANA_PATH
}),
]
urls.extend(metrics)
settings.Tornado_Setting["compress_response"] = True
settings.Tornado_Setting["job_worker"] = 10
utils.executor = ThreadPoolExecutor(
max_workers=settings.Tornado_Setting.get('job_worker', 10))
application = tornado.web.Application(urls, **settings.Tornado_Setting)
# 注册 restful api url
application_api_wrapper(application, authenticated,
need_auth=False) #restful api not need auth
# 注册nebula_strategy
from nebula.views.nebula_config import context as nebula_context
def load_event_schema():
events = [_.get_dict() for _ in nebula_context.nebula_events]
event_schema = dict()
for e in events:
properties = {p["name"]: p["type"] for p in e["properties"]}
event_schema[e["name"]] = properties
return event_schema
def load_variable_schema():
variables = [_.get_dict() for _ in nebula_context.nebula_variables]
variable_schema = dict()
for v in variables:
variable_schema[v["name"]] = {}
return variable_schema
# """定时任务处理，此后需要定时任务都可往里面加，无需加机器上的crontab"""
# scheduler = BackgroundScheduler()
# # 添加调度任务
# # 触发器选择 interval(间隔性)，间隔时长为 60 秒
# scheduler.add_job(scheduler_tasks[0], 'interval', seconds=60)
# # 启动调度任务
# scheduler.start()
# 定时60秒启动一次
tornado.ioloop.PeriodicCallback(scheduler_tasks[0],
callback_time=1000 * 60).start()
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(application, xheaders=True)
http_server.listen(str(port), settings.WebUI_Address)
click.echo("Nebula Web Start, Port:%s" % port)
tornado.ioloop.IOLoop.instance().start()