def run(self):
consumer = Consumer(self.kafka_config["config"].get(dict))
consumer.subscribe(self.kafka_config["topics"].get(list))
global logger, filter_config
while self._running():
message = consumer.poll(
timeout=self.kafka_config["poll_interval"].get(float))
if message is None:
continue
if message.error():
logger.error(f"Kafka error: {message.error()}")
continue
try:
msg = json.loads(message.value())
except Exception as e:
logger.error(f"Error decoding Kafka message: {e}")
continue
if not is_whitelisted(msg, filter_config):
continue
ioc = None
attr = msg.get("Attribute", None)
try:
ioc = attribute_to_stix2_indicator(attr,
msg.get("action",
None), logger)
except Exception as e:
logger.warn(f"Failed to parse MISP attribute {attr}: {e}")
if not ioc:
# the mapping function returns None e.g., in case a new MISP
# attribute is added without the `to_ids` flag enabled.
continue
self.inq.put(ioc)
def run(self):
global logger, filter_config
socket = zmq.Context().socket(zmq.SUB)
socket.connect(f"tcp://{self.zmq_config.host}:{self.zmq_config.port}")
# TODO: allow reception of more topics, i.e. handle events.
socket.setsockopt(zmq.SUBSCRIBE, b"misp_json_attribute")
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
while self._running():
socks = dict(poller.poll(timeout=1000))
if socket not in socks or socks[socket] != zmq.POLLIN:
continue
raw = socket.recv()
_, message = raw.decode("utf-8").split(" ", 1)
try:
msg = json.loads(message)
except Exception as e:
logger.error(f"Error decoding message {message}: {e}")
continue
if not is_whitelisted(msg, filter_config):
continue
ioc = None
attr = msg.get("Attribute", None)
try:
ioc = attribute_to_stix2_indicator(
attr, msg.get("action", None), logger
)
except Exception as e:
logger.warn(f"Failed to parse MISP attribute {attr}: {e}")
if not ioc:
# the mapping function returns None e.g., in case a new MISP
# attribute is added without the `to_ids` flag enabled.
continue
self.inq.put(ioc)
def snapshot(snapshot_request: SnapshotRequest, result_q: JoinableQueue):
global logger, misp, lock, filter_config
if snapshot_request.snapshot_type != MessageType.INDICATOR:
logger.debug("Sighting snapshot feature not yet implemented.")
return # TODO sighting snapshot not yet implemented
if not misp:
logger.debug(
"Cannot perform snapshot request. No MISP API connection.")
return
logger.info(
f"Executing intel snapshot for time delta {snapshot_request.snapshot}")
if not filter_config:
filter_config = [{}] # this empty whitelist results in a global query
# build queries for everything that is whitelisted
for fil in filter_config:
orgs = fil.get("orgs", [None])
types = fil.get("types", [None])
tags_query = misp.build_complex_query(
or_parameters=fil.get("tags", []))
if not tags_query:
tags_query = None # explicit None value
# By API design, orgs and types must be queried value-by-value
# None-values mean that all values are accepted
# https://pymisp.readthedocs.io/en/latest/_modules/pymisp/api.html#PyMISP.search
for (org, type_) in product(orgs, types):
lock.acquire()
data = misp.search(
org=org,
type_attribute=type_,
tags=tags_query,
controller="attributes",
to_ids=True,
date_from=datetime.now() - snapshot_request.snapshot,
)
lock.release()
if not data:
continue
for attr in data["Attribute"]:
try:
ioc = attribute_to_stix2_indicator(attr, "add", logger)
except Exception as e:
logger.warn(f"Failed to parse MISP attribute {attr}: {e}")
if ioc:
result_q.put(
SnapshotEnvelope(
snapshot_request.snapshot_type,
snapshot_request.snapshot_id,
ioc,
))
def test_valid_misp_attributes(self):
indicator = attribute_to_stix2_indicator(self.valid_misp_attribute,
"add", None)
self.assertEqual(indicator.type, self.indicator.type)
self.assertEqual(indicator.id, self.indicator.id)
self.assertEqual(indicator.created, self.indicator.created)
self.assertEqual(indicator.pattern, self.indicator.pattern)
single_value_valid_misp_attribute = self.valid_misp_attribute.copy()
single_value_valid_misp_attribute["type"] = "domain"
single_value_valid_misp_attribute["value"] = self.domain_ioc
single_value_valid_misp_attribute["value1"] = self.domain_ioc
single_value_valid_misp_attribute["value2"] = None
indicator = attribute_to_stix2_indicator(
single_value_valid_misp_attribute, "add", None)
self.assertEqual(indicator.type, self.indicator.type)
self.assertEqual(indicator.id, self.indicator.id)
self.assertEqual(indicator.created, self.ts)
self.assertEqual(indicator.pattern,
f"[domain-name:value = '{self.domain_ioc}']")
# test that no custom properties are set
for prop in indicator:
self.assertTrue(not prop.startswith("x_threatbus_"))
def test_valid_misp_attribute_removal(self):
valid_misp_attribute = self.valid_misp_attribute.copy()
valid_misp_attribute["to_ids"] = False
indicator = attribute_to_stix2_indicator(valid_misp_attribute, "edit", None)
self.assertEqual(indicator.x_threatbus_update, Operation.REMOVE.value)
