def test_messages(self):
with broker.Endpoint() as ep1, \
broker.Endpoint() as ep2, \
ep1.make_subscriber("/test") as s1:
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
msg0 = ("/test/1", ())
ep2.publish(*msg0)
# --messages-start
msg1 = ("/test/2", (1, 2, 3))
msg2 = ("/test/3", (42, "foo", {"a": "A", "b": ipaddress.IPv4Address('1.2.3.4')}))
ep2.publish_batch(msg1, msg2)
# --messages-end
msgs = s1.get(3)
self.assertFalse(s1.available())
self.assertEqual(msgs[0], msg0)
self.assertEqual(msgs[1], msg1)
self.assertEqual(msgs[2], msg2)
# These results are not (all) immutable: try modifying the third
# value (the dict) of the last message above.
dict_data = msgs[2][1][2]
self.assertEqual(len(dict_data), 2)
dict_data["c"] = "not immutable"
self.assertEqual(len(dict_data), 3)
def create_stores():
ep0 = broker.Endpoint()
s0 = ep0.make_subscriber("/test")
p = ep0.listen("127.0.0.1", 0)
ep1 = broker.Endpoint()
s1 = ep1.make_subscriber("/test")
es1 = ep1.make_status_subscriber()
ep1.peer("127.0.0.1", p)
ep2 = broker.Endpoint()
s2 = ep2.make_subscriber("/test")
es2 = ep2.make_status_subscriber()
ep2.peer("127.0.0.1", p)
# TODO: This doesn't work. Once it does, remove the event handshake.
# es1.get()
# es2.get()
ep0.publish("/test", "go-ahead")
s1.get()
s2.get()
####
m = ep0.attach_master("test", broker.Backend.Memory)
c1 = ep1.attach_clone("test")
c2 = ep2.attach_clone("test")
return (ep0, ep1, ep2, m, c1, c2)
def test_messages(self):
ep1 = broker.Endpoint()
ep2 = broker.Endpoint()
s1 = ep1.make_subscriber("/test")
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
msg0 = ("/test/1", ())
ep2.publish(*msg0)
# --messages-start
msg1 = ("/test/2", (1, 2, 3))
msg2 = ("/test/3", (42, "foo", {
"a": "A",
"b": ipaddress.IPv4Address('1.2.3.4')
}))
ep2.publish_batch(msg1, msg2)
# --messages-end
msgs = s1.get(3)
self.assertFalse(s1.available())
self.assertEqual(msgs[0], msg0)
self.assertEqual(msgs[1], msg1)
self.assertEqual(msgs[2], msg2)
ep1.shutdown()
ep2.shutdown()
def test_ping(self):
# --peer-start
ep1 = broker.Endpoint()
ep2 = broker.Endpoint()
s1 = ep1.make_subscriber("/test")
s2 = ep2.make_subscriber("/test")
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
# --peer-end
# --ping-start
ep2.publish("/test", ["ping"])
(t, d) = s1.get()
# t == "/test", d == ["ping"]
# --ping-end
self.assertEqual(t, "/test")
self.assertEqual(d[0], "ping")
ep1.publish(t, ["pong"])
(t, d) = s2.get()
self.assertEqual(t, "/test")
self.assertEqual(d[0], "pong")
ep1.shutdown()
ep2.shutdown()
def setup_peers(opts1=None,
opts2=None,
opts3=None,
opts4=None,
create_s1=True,
create_s2=True,
create_s3=True,
create_s4=True):
def cfg(opts):
return broker.Configuration(opts) if opts else broker.Configuration(
broker.BrokerOptions())
ep1 = broker.Endpoint(cfg(opts1))
ep2 = broker.Endpoint(cfg(opts2))
ep3 = broker.Endpoint(cfg(opts3))
ep4 = broker.Endpoint(cfg(opts4))
s1 = ep1.make_subscriber("/test/") if create_s1 else None
s2 = ep2.make_subscriber("/test/") if create_s2 else None
s3 = ep3.make_subscriber("/test/") if create_s3 else None
s4 = ep4.make_subscriber("/test/") if create_s4 else None
p2 = ep2.listen("127.0.0.1", 0)
p3 = ep3.listen("127.0.0.1", 0)
p4 = ep4.listen("127.0.0.1", 0)
# ep1 <-> ep2 <-> ep3 <-> ep4
ep1.peer("127.0.0.1", p2, 1.0)
ep2.peer("127.0.0.1", p3, 1.0)
ep3.peer("127.0.0.1", p4, 1.0)
return ((ep1, ep2, ep3, ep4), (s1, s2, s3, s4))
def test_ping(self):
# --peer-start
with broker.Endpoint() as ep1, \
broker.Endpoint() as ep2, \
ep1.make_subscriber("/test") as s1, \
ep2.make_subscriber("/test") as s2:
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
# --peer-end
# --ping-start
ep2.publish("/test", ["ping"])
(t, d) = s1.get()
# t == "/test", d == ["ping"]
# --ping-end
self.assertEqual(t, "/test")
self.assertEqual(d[0], "ping")
ep1.publish(t, ["pong"])
while True:
# This loop exists just for sake of test coverage for "poll()"
msgs = s2.poll()
if msgs:
self.assertEqual(len(msgs), 1)
(t, d) = msgs[0]
break;
time.sleep(0.1)
self.assertEqual(t, "/test")
self.assertEqual(d[0], "pong")
def listener():
ep = broker.Endpoint()
sub = ep.make_subscriber("edmand")
ep.listen("127.0.0.1", 9999)
total_time = 0
count = 0
while True:
(t, msg) = sub.get()
start = timeit.default_timer()
t = str(t)
ev = broker.bro.Event(msg)
if t == "edmand/packet_get":
raw_packet_queue.put_nowait(ev.args())
if t == "edmand/protocol_get":
raw_operation_queue.put_nowait(ev.args())
if t == "edmand/data_get":
raw_data_value_queue.put_nowait(ev.args())
if t == "edmand/bro_done":
ep.shutdown()
#print("Listener quit!")
if count != 0:
print("Listener time: " + str(total_time / count))
return
#print("got message")
total_time += timeit.default_timer() - start
count += 1
gevent.sleep(0)
def run(
config: DynaBox,
logging: DynaBox,
inq: JoinableQueue,
subscribe_callback: Callable,
unsubscribe_callback: Callable,
):
global logger, workers
logger = threatbus.logger.setup(logging, __name__)
assert plugin_name in config, f"Cannot find configuration for {plugin_name} plugin"
config = config[plugin_name]
broker_opts = broker.BrokerOptions()
broker_opts.forward = False
ep = broker.Endpoint(broker.Configuration(broker_opts))
ep.listen(config.host, config.port)
workers.append(
SubscriptionManager(config.module_namespace, ep, subscribe_callback,
unsubscribe_callback))
workers.append(BrokerReceiver(config.module_namespace, ep, inq))
workers.append(BrokerPublisher(config.module_namespace, ep))
for w in workers:
w.start()
logger.info("Zeek plugin started")
def run(
config: Subview,
logging: Subview,
inq: JoinableQueue,
subscribe_callback: Callable,
unsubscribe_callback: Callable,
):
global logger, workers
logger = threatbus.logger.setup(logging, __name__)
config = config[plugin_name]
try:
validate_config(config)
except Exception as e:
logger.fatal("Invalid config for plugin {}: {}".format(
plugin_name, str(e)))
host, port, namespace = (
config["host"].get(),
config["port"].get(),
config["module_namespace"].get(),
)
broker_opts = broker.BrokerOptions()
broker_opts.forward = False
ep = broker.Endpoint(broker.Configuration(broker_opts))
ep.listen(host, port)
workers.append(
SubscriptionManager(namespace, ep, subscribe_callback,
unsubscribe_callback))
workers.append(BrokerReceiver(namespace, ep, inq))
workers.append(BrokerPublisher(namespace, ep))
for w in workers:
w.start()
logger.info("Zeek plugin started")
def test_ssl_auth_success_self_signed(self):
cfg = broker.Configuration(broker.BrokerOptions())
cfg.openssl_certificate = data_path("cert.self-signed.pem")
cfg.openssl_key = data_path("key.self-signed.pem")
cfg.openssl_cafile = data_path("cert.self-signed.pem")
with broker.Endpoint(cfg) as ep1, \
broker.Endpoint(cfg) as ep2, \
ep1.make_subscriber("/test") as s1, \
ep2.make_subscriber("/test") as s2:
port = ep1.listen("127.0.0.1", 0)
r = ep2.peer("127.0.0.1", port, 0)
self.assertEqual(r, True)
self.check_ping(ep1, s1, ep2, s2)
def XXXtest_ssl_auth_failure_ca_pw(self):
cfg = broker.Configuration(broker.BrokerOptions())
cfg.openssl_certificate = data_path("cert.1.pem")
cfg.openssl_key = data_path("key.1.enc.pem")
cfg.openssl_cafile = data_path("ca.pem")
cfg.openssl_passphrase = "WRONG PASSWORD"
with broker.Endpoint(cfg) as ep1, \
broker.Endpoint(cfg) as ep2:
port = ep1.listen("127.0.0.1", 0)
# TODO: This correctly generates an exception in CAF, for which I
# don't know where to catch it.
r = ep2.peer("127.0.0.1", port, 0)
self.assertEqual(r, False)
def _send_event_init(node, event, args, result_event, topic):
host = node.addr
endpoint = broker.Endpoint()
subscriber = endpoint.make_subscriber(topic)
status_subscriber = endpoint.make_status_subscriber(True)
endpoint.peer(host, node.getPort(), 1)
tries = 0
while True:
msgs = status_subscriber.get(1, 1)
for msg in msgs:
if isinstance(msg, broker.Status):
if msg.code() == broker.SC.PeerAdded:
ev = broker.zeek.Event(event, *args)
endpoint.publish(topic + "/" + repr(msg.context()), ev)
logging.debug("broker: %s(%s) to node %s", event,
", ".join(args), node.name)
return (True, endpoint, subscriber)
tries += 1
if tries > config.Config.commtimeout:
return (False, "time-out", None)
def send_generic(topic, items):
ep = broker.Endpoint()
status_subscriber = ep.make_status_subscriber(True)
ep.peer("127.0.0.1", 47761)
# blocking operation. wait until you get a status.
status = status_subscriber.get()
if type(status) != broker.Status or status.code() != broker.SC.PeerAdded:
print("peering with remote machine failed")
sys.exit(1)
for i in range(items):
data = {
"indicator": "example.com",
"intel_type": "DOMAIN",
}
event = broker.zeek.Event("intel", datetime.now(), i, data, "ADD")
# Threat Bus will pickup the event type and hence forward on a different
# topic.
ep.publish(topic, event)
## apparently the receiver will not receive everything, if the sending process exits too early. Thus we wait here (just for the demo sake)
time.sleep(1)
def test_ping(self):
ep = broker.Endpoint()
sub = ep.make_subscriber("/test")
port = ep.listen("127.0.0.1", 0)
p = multiprocessing.Process(target=RunZeek, args=(ZeekPing, port))
p.daemon = True
p.start()
for i in range(0, 6):
(t, msg) = sub.get()
ev = broker.zeek.Event(msg)
(s, c) = ev.args()
expected_arg = "x" + "Xx" * i
if i == 5:
expected_arg = expected_arg.encode('utf-8') + b'\x82'
self.assertEqual(ev.name(), "ping")
self.assertEqual(s, expected_arg)
self.assertEqual(c, i)
if i < 3:
ev = broker.zeek.Event("pong", s + "X", c)
elif i < 5:
ev = broker.zeek.Event("pong", s.encode('utf-8') + b'X', c)
else:
ev = broker.zeek.Event("pong", 'done', c)
ep.publish("/test", ev)
ep.shutdown()
def run(config, logging, inq, subscribe_callback, unsubscribe_callback):
logger = threatbus.logger.setup(logging, __name__)
config = config[plugin_name]
try:
validate_config(config)
except Exception as e:
logger.fatal("Invalid config for plugin {}: {}".format(plugin_name, str(e)))
host, port, namespace = (
config["host"].get(),
config["port"].get(),
config["module_namespace"].get(),
)
broker_opts = broker.BrokerOptions()
broker_opts.forward = False
ep = broker.Endpoint(broker.Configuration(broker_opts))
ep.listen(host, port)
logger.info(f"Broker: endpoint listening - {host}:{port}")
threading.Thread(
target=listen, args=(logger, namespace, ep, inq), daemon=True
).start()
threading.Thread(
target=manage,
args=(logger, namespace, ep, subscribe_callback, unsubscribe_callback),
daemon=True,
).start()
threading.Thread(target=publish, args=(logger, namespace, ep), daemon=True).start()
logger.info("Zeek plugin started")
def __init__(self, cfg):
self.config = cfg
# Broker Configuration
bcfg = broker.Configuration()
# (Optional) SSL Configuration
if cfg.BrokerSSLCAFile:
bcfg.openssl_cafile = cfg.BrokerSSLCAFile # Path to CA file
if cfg.BrokerSSLCAPath:
bcfg.openssl_capath = cfg.BrokerSSLCAPath # Path to directory with CA files
if cfg.BrokerSSLCertificate:
bcfg.openssl_certificate = cfg.BrokerSSLCertificate # Own certificate
if cfg.BrokerSSLKeyFile:
bcfg.openssl_key = cfg.BrokerSSLKeyFile # Own key
self.endpoint = broker.Endpoint(bcfg)
# Status Subscriber
self.status_queue = self.endpoint.make_status_subscriber(True)
# Message Subscribers
self.log_queue = self.endpoint.make_subscriber("logs")
self.file_queue = self.endpoint.make_subscriber("files")
self.elastic = get_instance(self.config.ElasticIP,
self.config.ElasticPort)
def __init__(self, queue, host, port, acld_hosts, acld_port, log_user,
log_host):
self.logger = logging.getLogger("brokerlisten")
self.queuename = queue
self.epl = broker.Endpoint()
self.epl.listen(host, port)
self.status_subscriber = self.epl.make_status_subscriber(True)
self.subscriber = self.epl.make_subscriber(self.queuename)
# Create a random list of host port pairs
self.acld_hosts = []
for host in acld_hosts:
self.acld_hosts.append(hostportpair(host, acld_port))
random.shuffle(self.acld_hosts)
self.ident = '{%s@%s}' % (log_user, log_host)
self.remote_ident = '?'
self.sock = None
self.waiting = {}
self.buffer = ''
self.acldstring = False
self.acldcmd = {}
# try to connect to acld
self.connect()
def test_status_subscriber(self):
# --status-start
with broker.Endpoint() as ep1, \
broker.Endpoint() as ep2, \
ep1.make_status_subscriber(True) as es1, \
ep2.make_status_subscriber(True) as es2:
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
st1 = es1.get()
st2 = es2.get()
# st1.code() == broker.SC.PeerAdded, st2.code() == broker.SC.PeerAdded
# --status-end
self.assertEqual(st1.code(), broker.SC.PeerAdded)
self.assertEqual(st1.context().network.get().address, "127.0.0.1")
self.assertEqual(st2.code(), broker.SC.PeerAdded)
self.assertEqual(st2.context().network.get().address, "127.0.0.1")
def test_publisher(self):
with broker.Endpoint() as ep1, \
broker.Endpoint() as ep2, \
ep1.make_subscriber("/test") as s1, \
ep2.make_publisher("/test") as p2:
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
p2.publish([1, 2, 3])
p2.publish_batch(["a", "b", "c"], [True, False])
msgs = s1.get(3)
self.assertFalse(s1.available())
self.assertEqual(msgs[0], ("/test", (1, 2, 3)))
self.assertEqual(msgs[1], ("/test", ("a", "b", "c")))
self.assertEqual(msgs[2], ("/test", (True, False)))
def __init__(self, queue, host, port):
self.queuename = queue
self.epl = broker.Endpoint()
self.epl.listen(host, port)
self.status_subscriber = self.epl.make_status_subscriber(True)
self.subscriber = self.epl.make_subscriber(self.queuename)
logger.debug("Set up listener for " + host + ":" + str(port) + " (" +
queue + ")")
def test_publisher(self):
ep1 = broker.Endpoint()
ep2 = broker.Endpoint()
s1 = ep1.make_subscriber("/test")
p2 = ep2.make_publisher("/test")
port = ep1.listen("127.0.0.1", 0)
ep2.peer("127.0.0.1", port, 1.0)
p2.publish([1, 2, 3])
p2.publish_batch(["a", "b", "c"], [True, False])
msgs = s1.get(3)
self.assertFalse(s1.available())
self.assertEqual(msgs[0], ("/test", [1, 2, 3]))
self.assertEqual(msgs[1], ("/test", ["a", "b", "c"]))
self.assertEqual(msgs[2], ("/test", [True, False]))
ep1.shutdown()
ep2.shutdown()
def test_basic(self):
# --master-start
with broker.Endpoint() as ep1, \
ep1.attach_master("test", broker.Backend.Memory) as m:
m.put("key", "value")
x = m.get("key")
# x == "value"
# --master-end
self.assertEqual(x, "value")
self.assertEqual(m.name(), "test")
def test_ssl_auth_failure_no_ssl(self):
cfg1 = broker.Configuration(broker.BrokerOptions())
cfg1.openssl_certificate = data_path("cert.1.pem")
cfg1.openssl_key = data_path("key.1.pem")
cfg1.openssl_cafile = data_path("ca.pem")
cfg2 = broker.Configuration(broker.BrokerOptions())
with broker.Endpoint(cfg1) as ep1, \
broker.Endpoint(cfg2) as ep2:
port = ep1.listen("127.0.0.1", 0)
r = ep2.peer("127.0.0.1", port, 0)
self.assertEqual(r, False)
with broker.Endpoint(cfg2) as ep1, \
broker.Endpoint(cfg1) as ep2:
port = ep1.listen("127.0.0.1", 0)
r = ep2.peer("127.0.0.1", port, 0)
self.assertEqual(r, False)
def test_basic(self):
# --master-start
ep1 = broker.Endpoint()
m = ep1.attach_master("test", broker.Backend.Memory)
m.put("key", "value")
x = m.get("key")
# x == "value"
# --master-end
self.assertEqual(x, "value")
self.assertEqual(m.name(), "test")
ep1.shutdown()
def __init__(self, *args, **kwargs):
super(BroController, self).__init__(*args, **kwargs);
self.dpset = kwargs['dpset']
self.data = {}
self.data['dpset'] = self.dpset
# store DPID->name mapping. We create the mapping implicitely
# for all flow_clear and flow_mod events that we get and use
# it for the returning flow_removed events
self.dpids = {}
self.epl = broker.Endpoint()
def __init__(self):
""" Manage the connection the broker network """
self.mc_id = "mc-" + str(random.randint(100000, 9999999))
self.endpoint = broker.Endpoint()
self.answerQueue = self.endpoint.make_subscriber("answer")
self.logsQueue = self.endpoint.make_subscriber("logs")
#increments if new peering is created
#used to identify peerings
self.connection_counter = 0
#dictionary of active peerings
self.conncection_dic = {}
def broker_receiver(event_queue):
## demo receiver that is subscribed to the topic "demo/multi-processing-python"
ep = broker.Endpoint()
subscriber = ep.make_subscriber("demo/multi-processing-python")
ep.listen("127.0.0.1", 9999)
print("endpoint listening...")
while True:
fd_sets = select.select([subscriber.fd()], [], [])
if not fd_sets[0]:
print("boom. this is the end.")
(_, data) = subscriber.get()
event = broker.bro.Event(data)
event_queue.put([event.name(), event.args()])
def connect_bro(self):
self.logger.info("connect to bro on port %s", self.bro_port)
self.ep = broker.Endpoint()
self.sub = self.ep.make_subscriber(BroHttpDriver.EVENT_TOPIC)
self.ss = self.ep.make_status_subscriber(True)
self.ep.peer("127.0.0.1", self.bro_port)
self.logger.info("connect to bro on port 1 %s", self.bro_port)
st = self.ss.get()
self.logger.info("connect to bro on port 2 %s", self.bro_port)
if not (type(st) == broker.Status
and st.code() == broker.SC.PeerAdded):
self.logger.info("connect to bro failed")
raise RuntimeError("connect to bro failed")
self.logger.info("connect to bro successed")
def receive(items, topic="stix2/indicator", time_delta=timedelta(days=0)):
host, port = "127.0.0.1", 47761
ep = broker.Endpoint()
ep.peer(host, port)
p2p_topic = subscribe(ep, topic, time_delta)
subscriber = ep.make_subscriber(p2p_topic)
for _ in range(items):
fd_sets = select.select([subscriber.fd()], [], [])
if not fd_sets[0]:
print("Receiving data from Threat Bus failed.")
return
_, data = subscriber.get()
yield broker.zeek.Event(data)
unsubscribe(ep, p2p_topic)
def broker_receiver():
## demo receiver that is subscribed to the topic "demo/threaded-simple"
ep = broker.Endpoint()
subscriber = ep.make_subscriber("demo/threaded-simple")
ep.listen("127.0.0.1", 9999)
print("endpoint listening...")
while True:
fd_sets = select.select([subscriber.fd()], [], [])
if not fd_sets[0]:
print("boom. this is the end.")
(topic, data) = subscriber.get()
received_event = broker.bro.Event(data)
print("received on topic: {} event name: {} content: {}".format(
topic, received_event.name(), received_event.args()))
