def start(self):
if self.running:
return
self.logger.info("start tshark driver on interface %s for ports %s, with bpf filter %s", self.interface,
self.ports, self.bpf_filter)
self.running = True
self.data_mr = MetricsRecorder("sniffer.driver.data")
self.error_mr = MetricsRecorder("sniffer.driver.error")
port_filter = " or ".join(["tcp port {}".format(port) for port in self.ports])
if self.bpf_filter:
port_filter = "({}) and ({})".format(port_filter, self.bpf_filter)
tshark_home = get_tshark_home()
if not tshark_home:
raise RuntimeError("tshark is not find")
self.logger.info("find tshark at %s", tshark_home)
command = (is_linux() and "sudo " or "") + """%(tshark_home)s/tshark -o ssl.desegment_ssl_application_data:TRUE
-o ssl.desegment_ssl_records:TRUE
-o ssl.keys_list:"0.0.0.0","443","http","/home/threathunter/private.key"
-f "%(port_filter)s"
-i %(interface)s
-Y "http.request or http.response"
-T fields -Eseparator=/t
-e http
-e http.request
-e ip.src
-e tcp.srcport
-e ip.dst
-e tcp.dstport
-e http.request.method
-e http.host
-e http.request.uri
-e http.request.full_uri
-e http.user_agent
-e http.content_length
-e http.content_type
-e http.response.code
-e http.response.phrase
-e http.content_encoding
-e http.cookie
-e http.set_cookie
-e http.referer
-e data.data
-e text
""" % ({"tshark_home": tshark_home, "port_filter": port_filter,
"interface": self.interface})
environments = dict()
environments["PCAP_PF_RING_CLUSTER_ID"] = "14"
environments["PCAP_PF_RING_APPNAME"] = "tshark-" + self.interface
environments["PCAP_PF_RING_USE_CLUSTER_PER_FLOW_4_TUPLE"] = "1"
environments["LD_LIBRARY_PATH"] = "/usr/local/lib64"
self.logger.info("start tshark command: %s", command)
self.sub_task = pexpect.spawn(command, env=environments, timeout=3600)
import atexit
atexit.register(self.stop)
# establish client
gevent.sleep(2)
self.client_task = run_in_thread(self.process_input)
return
def notice_clean():
global logger
logger_name = "nebula.notice.cleaner"
start_time = time.time()
error_type = None
job_name = "Clean table notice"
try:
logger = utils.init_env(logger_name)
notice_clean_recorder = MetricsRecorder("cronjob.notice_clean",
expire=86400* 60, #2month
interval=300, #5min to merge
type="sum",
db="nebula.offline")
cj = NoticeCleanCronJob()
notice_clean_recorder.record(1, {"status":"run"})
cj.start()
status = "success"
except Exception as e:
logger.exception(e.message)
status = "fail"
error_type = e.message
finally:
costs = (time.time() - start_time)/ 60.0
logger.info("Cronjob(%s) start at %s has been %s , costs %s min.", job_name, start_time, status, costs)
notice_clean_recorder.record(1, {"status":status,
"error_type":error_type,
"costs": costs})
# wait for metrics to write.
gevent.sleep(60)
def __init__(self, id, parser, driver, cpu=None, is_process=True):
self.parser = parser
self.driver = driver
self.id = id
self._running = False
self._rpc_task = None
self._events_task = None
self._health_task = None
self.queue = gevent.queue.Queue(maxsize=10000)
self.cpu = cpu
self.is_process = is_process
self.logger = settings.init_logging("main.{}".format(self.id))
self.error_mr = MetricsRecorder("sniffer.main.error")
self.msg_mr = MetricsRecorder("sniffer.main.msg")
self.event_mr = MetricsRecorder("sniffer.main.event")
self.rpc_mr = MetricsRecorder("sniffer.main.rpc")
self.main_mr = MetricsRecorder("sniffer.main.loop")
self.urltree = URLTree()
def notice_stat(timestamp):
global logger
logger_name = "nebula.notice_stat.writer"
job_name = "generate notice stat"
error_type = None
start_time = time.time()
# 获取需要转换notice -> notice_stat 的时间戳, 如果不指定，默认转换的notice的时间范围是上个小时内
if not timestamp:
timestamp = utils.get_last_hour()
t = datetime.strptime(timestamp, settings.LogPath_Format)
settings.Working_TS = time.mktime((t.year, t.month, t.day, t.hour, t.minute, t.second, 0, 0, 0))
settings.Working_DAY = int(time.mktime((t.year, t.month, t.day, 0, 0, 0, 0, 0, 0)))
click.echo(u"所使用的工作小时的时间戳是:%s, 既:%s" %
(settings.Working_TS, datetime.fromtimestamp(settings.Working_TS)))
click.echo(u"所处的日期是%s, 既:%s" %
(settings.Working_DAY, datetime.fromtimestamp(settings.Working_DAY*1.0)))
try:
logger = utils.init_env(logger_name)
notice_stat_recorder = MetricsRecorder("cronjob.notice_stat",
expire=86400* 60, #2month
interval=300, #5min to merge
type="sum",
db="nebula.offline")
utils.get_strategies_weigh()
cj = NoticeStatCronJob()
notice_stat_recorder.record(1, {"status":"run", "workingts":settings.Working_TS})
cj.start()
status = "success"
except Exception as e:
logger.exception(traceback.format_exc())
status = "fail"
error_type = e.message
finally:
costs = (time.time() - start_time)/ 60.0
logger.info("Cronjob(%s) working ts: %s has been %s, costs: %s min.", job_name, settings.Working_TS, status, costs)
notice_stat_recorder.record(1, {"status":status,
"workingts":settings.Working_TS,
"error_type":error_type})
# wait for metrics to write.
gevent.sleep(60)
conditions.append(get_op_tip(left, ops[op], right))
tip += '{}s内，{}{}的总数等于{};'.format(interval, source_event, conditions and '在满足{}的情况下'.format(",".join(conditions)) or "", v)
elif left.subtype == "getvariable":
counter = dbcontext.nebula_ui_variables_dict.get(tuple(left.variable))
counter_name = counter.name
v = values.get(counter_name)
if v:
tip += '{}等于"{}";'.format(counter_name, v)
else:
continue
tip += "满足策略{}".format(name)
# print tip
return tip
metrics_recorder_stats = MetricsRecorder("web.notices.stats", expire=86400*60, interval=300, type="sum", db="default")
def add_metrics(notice):
metrics_recorder_stats.record(1, {"test": 1 if notice.test else 0})
def process_notify(event):
if not event:
return
try:
data = {
"timestamp": event.timestamp,
"key": event.key,
"scene_name": event.property_values["sceneName"],
def start(self):
self.logger.info("start syslog driver")
self.data_mr = MetricsRecorder("sniffer.driver.data")
self.error_mr = MetricsRecorder("sniffer.driver.error")
self.server.start()
class ServiceClient(object):
send_metrics = MetricsRecorder("babel.client.sendcount", db="fx", type="count")
costrange_metrics = MetricsRecorder("babel.client.costrange", db="fx", type="count")
cost_max_metrics = MetricsRecorder("babel.client.cost.max", db="fx", type="max")
cost_min_metrics = MetricsRecorder("babel.client.cost.min", db="fx", type="min")
cost_avg_metrics = MetricsRecorder("babel.client.cost.avg", db="fx", type="avg")
timeout_metrics = MetricsRecorder("babel.client.timeout", db="fx", type="count")
error_metrics = MetricsRecorder("babel.client.error", db="fx", type="count")
def __init__(self, service_meta, **kwargs):
self.raise_if_connect_error = kwargs.get('raise_if_connect_error',True) #add by wxt 2015-12-16 如果初始化失败是否raise异常
self.running = False
try:
self.service_meta = service_meta
# atexit
if self.service_meta.callmode == "notify":
atexit.register(self.batch_notify_flush)
self.client_id = kwargs.get("client_id")
if not self.client_id:
self.client_id = gen_uuid()
self.client_id = '_client.{}.{}'.format(self.service_meta.name,self.client_id)
if service_meta.serverimpl == "rabbitmq":
from . import babelrabbitmq
self.impl = babelrabbitmq
elif service_meta.serverimpl == "redis":
from . import babelredis
self.impl = babelredis
else:
raise RuntimeError("serverimpl {} not implemented yet".format(service_meta.serverimpl))
if "sdc" not in self.service_meta.options:
raise RuntimeError("sdc not in service meta")
self.sdc = self.service_meta.options.get("sdc", "")
if "cdc" not in self.service_meta.options:
raise RuntimeError("cdc not in service meta")
self.cdc = self.service_meta.options.get("cdc", "")
if "," in self.service_meta.options["sdc"] and self.service_meta.callmode != "notify":
raise RuntimeError("only notify supports multiple data center")
self._sender = self.impl.get_client_sender(service_meta, **kwargs)
if service_meta.callmode != "notify":
# need a queue to receive response
self._receiver = self.impl.get_client_receiver(service_meta, self.client_id, **kwargs)
else:
self._receiver = None
if service_meta.coder != "mail":
raise RuntimeError("coder {} is not supported yet".format(service_meta.coder))
self.requestid_base = AtomicLong(0)
self.request_cache = RequestCache()
self.running = True
self.response_task = None
self.metrics_tags = {
"service": service_meta.name,
"delivery": service_meta.delivermode,
"call": service_meta.callmode,
"impl": service_meta.serverimpl,
"clientid": self.client_id,
"sdc": self.sdc,
"cdc": self.cdc
}
# caching requests for batch_notify
self.batch_cache = list()
self.running = True
except Exception,e:
if self.raise_if_connect_error:
print e
raise RuntimeError('babel connect error')
def start(self):
self.logger.info("start redis list driver")
self.running = True
self.data_mr = MetricsRecorder("sniffer.driver.data")
self.error_mr = MetricsRecorder("sniffer.driver.error")
task = gevent.spawn(self.redis_loop)
import utils
import settings
from model import Incident
logger = logging.getLogger("nebula.dbwriter.incident")
Test_Get_Event_Count = 0
Test_Get_All_Event_Time = None
Test_Save_All_Time = None
Not_Null_Cols = ("ip", "start_time", "strategies", "hit_tags", "risk_score", "uri_stems",\
"hosts", "dids", "associated_users")
Incident_Metrics_Recorder = MetricsRecorder("store.incident",
expire=86400* 60, #2month
interval=300, #5min to merge
type="sum",
db="nebula.offline")
def add_success_metrics():
add_incident_metrics("success")
def add_receive_metrics():
add_incident_metrics("receive")
def add_fail_metrics():
add_incident_metrics("fail")
def add_incident_metrics(status):
Incident_Metrics_Recorder.record(1, {"status":status})
def start(self):
if self.running:
return
self.running = True
self.logger.info('bro starting:{}'.format(self.running))
# metrics should initialize in its own process
self.data_mr = MetricsRecorder("sniffer.driver.data")
self.error_mr = MetricsRecorder("sniffer.driver.error")
# establish bro
if self.embedded_bro:
if not self.interface:
self.add_error_metrics("invalid params")
raise RuntimeError("null interface")
if not self.ports:
self.add_error_metrics("invalid params")
raise RuntimeError("null ports")
self.logger.info(
"trying to start bro driver on interface %s for ports %s",
self.interface, self.ports)
tmp_bro_file_name = os.path.join(
"tmp", "worker-{}-{}".format(self.interface, self.idx))
out = file(tmp_bro_file_name, "w")
print >> out, "@load policy/frameworks/control/controllee"
print >> out, "@load policy/misc/loaded-scripts.bro"
print >> out, "redef Control::controllee_listen = F;"
print >> out, "redef Broker::default_listen_address = \"127.0.0.1\";"
print >> out, "redef Broker::default_port = %s/tcp;" % self.bro_port
ports_str = "".join("{}/tcp,".format(_) for _ in self.ports)
print >> out, "const ports = {"
print >> out, ports_str
print >> out, "};"
print >> out, "redef likely_server_ports += { ports };"
out.close()
executable = os.path.join(self.bro_home, "bin/bro")
#script = os.path.join(self.bro_home, "share/bro/base/protocols/http/main.bro")
script = os.path.join(settings.Conf_Sniffer_Path, "http.bro")
environments = dict()
environments["PCAP_PF_RING_CLUSTER_ID"] = "13"
environments["PCAP_PF_RING_APPNAME"] = "bro-" + self.interface
environments["PCAP_PF_RING_USE_CLUSTER_PER_FLOW_4_TUPLE"] = "1"
self.logger.info('init bro, bro home is {}'.format(self.bro_home))
self.logger.info(
'init bro, bro executable is {}'.format(executable))
self.logger.info('init bro, bro interface is {}'.format(
self.interface))
self.logger.info('init bro, bro bpf filter is {}'.format(
self.bpf_filter))
self.logger.info(
'init bro, bro temp file is {}'.format(tmp_bro_file_name))
self.logger.info('init bro, bro script is {}'.format(script))
if self.bpf_filter:
# use bpf filter
self.sub_task = subprocess.Popen([
executable, "-C", "-b", "-i", self.interface, "-f",
self.bpf_filter, tmp_bro_file_name, script
],
shell=False,
preexec_fn=os.setsid,
stderr=sys.stderr,
stdout=sys.stdout,
env=environments)
else:
self.sub_task = subprocess.Popen([
executable, "-C", "-b", "-i", self.interface,
tmp_bro_file_name, script
],
shell=False,
preexec_fn=os.setsid,
stderr=sys.stderr,
stdout=sys.stdout,
env=environments)
atexit.register(self.stop)
# establish client
gevent.sleep(5)
self.connect_bro()
self.config_bro()
self.client_task = gevent.spawn(self.process_input)
self.logger.info("driver start")
self.client_task.start()
return
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import json
from tornado.web import RequestHandler
from threathunter_common.metrics.metricsrecorder import MetricsRecorder
from threathunter_common.metrics.metricsagent import get_latency_str_for_millisecond
import settings
cost_range_metrics = MetricsRecorder(
"web.api.cost.range", db="default", type="count", expire=86400 * 7, interval=60)
cost_max_metrics = MetricsRecorder(
"web.api.cost.max", db="default", type="max", expire=86400 * 7, interval=60)
cost_min_metrics = MetricsRecorder(
"web.api.cost.min", db="default", type="min", expire=86400 * 7, interval=60)
cost_avg_metrics = MetricsRecorder(
"web.api.cost.avg", db="default", type="avg", expire=86400 * 7, interval=60)
user_access_metrics = MetricsRecorder(
"web.api.user.count", db="default", type="count", expire=86400 * 7, interval=60)
class BaseHandler(RequestHandler):
def data_received(self, chunk):
pass
def get_current_user(self):
return self.get_secure_cookie("user")
def start(self):
self.logger.info("start logstash driver")
self.data_mr = MetricsRecorder("sniffer.driver.data")
self.error_mr = MetricsRecorder("sniffer.driver.error")
self.server.start_running()
class ServiceServer(object):
processed_metrics = MetricsRecorder("babel.server.processcount",
type="count",
db="fx",
interval=5)
cost_range_metrics = MetricsRecorder("babel.server.costrange",
type="count",
db="fx",
interval=5)
cost_avg_metrics = MetricsRecorder("babel.server.cost.avg",
type="avg",
db="fx",
interval=5)
cost_max_metrics = MetricsRecorder("babel.server.cost.max",
type="max",
db="fx",
interval=5)
cost_min_metrics = MetricsRecorder("babel.server.cost.min",
type="min",
db="fx",
interval=5)
error_metrics = MetricsRecorder("babel.server.error",
type="count",
db="fx",
interval=5)
def __init__(self, service_meta, func=None, workers=5, **kwargs):
self.service_meta = service_meta
self.server_id = kwargs.get("server_id")
self.func = func
self.sender_cache = dict()
self.kwargs = kwargs
self.workers = workers
if not self.server_id:
self.server_id = gen_uuid()
if service_meta.serverimpl == "rabbitmq":
from . import babelrabbitmq
self.impl = babelrabbitmq
elif service_meta.serverimpl == "redis":
from . import babelredis
self.impl = babelredis
else:
raise RuntimeError("serverimpl {} not implemented yet".format(
service_meta.serverimpl))
if "sdc" not in self.service_meta.options:
raise RuntimeError("sdc not in service meta")
self.sdc = self.service_meta.options.get("sdc", "")
if "," in self.service_meta.options["sdc"]:
raise RuntimeError("server should only have one dc")
if service_meta.coder != "mail":
raise RuntimeError("coder {} is not supported yet".format(
service_meta.coder))
self._receiver = self.impl.get_server_receiver(service_meta, **kwargs)
#if not func:
#raise RuntimeError("the service implementation should not by empty")
if self.workers > 1:
self.worker_pool = ThreadPool(processes=self.workers)
self.running = True
self.metrics_tags = {
"service": service_meta.name,
"delivery": service_meta.delivermode,
"call": service_meta.callmode,
"impl": service_meta.serverimpl,
"serverid": self.server_id,
"sdc": self.sdc
}
def _get_sender(self, cdc, client_id):
key = "{}.{}".format(cdc, client_id)
result = self.sender_cache.get(key)
if result:
return result
result = self.impl.get_server_sender(cdc, client_id, **self.kwargs)
self.sender_cache[key] = result
return result
def start(self, func=None, sync=False):
if not self.func:
self.func = func
self.running = True
self._receiver.start_consuming()
if sync:
self.accept()
else:
self.accept_task = run_in_thread(self.accept)
def start_sync(self, func=None):
self.start(func=func, sync=True)
def close(self):
if not self.running:
return
self.running = False
self._receiver.stop_consuming()
if self.accept_task:
self.accept_task.join()
self.accept_task = None
if self.worker_pool:
self.worker_pool.close()
self.worker_pool.join()
self.worker_pool = None
if self._receiver:
self._receiver.close()
self._receiver = None
def work(self, *args, **kwargs):
if self.workers == 1:
# run in this thread
self.process_mail(*args, **kwargs)
else:
self.worker_pool.apply(self.process_mail, args, kwargs)
def accept(self):
while self.running:
try:
request_mail = self._receiver.get(True, 0.5)
request_mail = Mail.from_json(request_mail)
accept_ts = millis_now()
self.worker_pool.apply(self.process_mail,
[(request_mail, accept_ts)])
except Queue.Empty:
pass
except Exception as error:
tags = {"type": "accept"}
tags.update(self.metrics_tags)
ServiceServer.error_metrics.record(1, tags)
print error
self._receiver.close()
for request_mail in self._receiver.dump_cache():
self.worker_pool.apply(self.process_mail, request_mail)
def process_mail(self, args):
try:
cdc = ""
client_id = ""
try:
request_mail, accept_ts = args
cdc = request_mail.get_header("cdc", "")
client_id = request_mail.f
process_start_ts = millis_now()
tags = {
"type": "accept2process",
"cdc": cdc,
"clientid": client_id
}
tags.update(self.metrics_tags)
cost = process_start_ts - accept_ts
ServiceServer.cost_avg_metrics.record(cost, tags)
ServiceServer.cost_max_metrics.record(cost, tags)
ServiceServer.cost_min_metrics.record(cost, tags)
events = extract_data_from_mail(request_mail)
if isinstance(events, list):
for e in events:
result = self.func(e)
else:
result = self.func(events)
finish_func_ts = millis_now()
tags = {
"type": "invokefunction",
"cdc": cdc,
"clientid": client_id
}
tags.update(self.metrics_tags)
cost = finish_func_ts - process_start_ts
ServiceServer.cost_avg_metrics.record(cost, tags)
ServiceServer.cost_max_metrics.record(cost, tags)
ServiceServer.cost_min_metrics.record(cost, tags)
success = True
except Exception, err:
tags = {
"type": "invokefunction",
"cdc": cdc,
"clientid": client_id
}
tags.update(self.metrics_tags)
ServiceServer.error_metrics.record(1, tags)
success = False
result = str(err)
if self.service_meta.callmode != "notify":
sender = self._get_sender(cdc, client_id)
response_mail = Mail.new_mail(self.server_id, client_id,
request_mail.requestid)
populate_data_into_mail(response_mail, result)
sender.send(response_mail.get_json(), False, 1)
after_response_ts = millis_now()
tags = {
"type": "sendresponse",
"cdc": cdc,
"clientid": client_id
}
tags.update(self.metrics_tags)
cost = after_response_ts - finish_func_ts
ServiceServer.cost_avg_metrics.record(cost, tags)
ServiceServer.cost_max_metrics.record(cost, tags)
ServiceServer.cost_min_metrics.record(cost, tags)
except Exception as error:
tags = {"type": "sendresponse", "cdc": cdc, "clientid": client_id}
tags.update(self.metrics_tags)
ServiceServer.error_metrics.record(1, tags)
success = False
