async def live_match_vast(vast_binary: str, vast_endpoint: str,
sightings_queue: asyncio.Queue):
"""
Starts a VAST matcher. Enqueues all matches from VAST to the
sightings_queue.
@param vast_binary The VAST binary command to use with PyVAST
@param vast_endpoint The endpoint of a running VAST node
@param sightings_queue The queue to put new sightings into
"""
global logger, matcher_name
vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
proc = await vast.matcher().attach().json(matcher_name).exec()
# returncode is None as long as the process did not terminate yet
while proc.returncode is None:
data = await proc.stdout.readline()
if not data:
if not await vast.test_connection():
logger.error("Lost connection to VAST, cannot live-match")
# TODO reconnect
continue
vast_sighting = data.decode("utf-8").rstrip()
sighting = matcher_result_to_sighting(vast_sighting)
if not sighting:
logger.error(
f"Cannot parse sighting-output from VAST: {vast_sighting}")
continue
g_live_matcher_sightings.inc()
logger.info(f"Got a new sighting from VAST")
await sightings_queue.put(sighting)
stderr = await proc.stderr.read()
if stderr:
logger.error("VAST matcher process exited with message: {}".format(
stderr.decode()))
logger.critical("Unexpected exit of VAST matcher process.")
async def retro_match_vast(
vast_binary: str,
vast_endpoint: str,
retro_match_max_events: int,
retro_match_timeout: float,
indicator: Indicator,
sightings_queue: asyncio.Queue,
):
"""
Turns the given STIX-2 Indicator into a valid VAST query and forwards all
query results (sightings) to the sightings_queue.
@param vast_binary The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node ('host:port')
@param retro_match_max_events Max amount of retro match results
@param retro_match_timeout Interval after which to terminate the retro-query
@param indicator The STIX-2 Indicator to query VAST for
@param sightings_queue The queue to put new sightings into
"""
start = time.time()
query = indicator_to_vast_query(indicator)
if not query:
return
global logger, max_open_tasks
async with max_open_tasks:
vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
kwargs = {}
if retro_match_max_events > 0:
kwargs["max_events"] = retro_match_max_events
proc = await vast.export(**kwargs).json(query).exec()
retro_result = None
try:
retro_result = await asyncio.wait_for(
proc.communicate(),
timeout=retro_match_timeout if retro_match_timeout > 0 else None,
)
except asyncio.TimeoutError:
proc.terminate()
logger.error(
f"Timeout after {retro_match_timeout}s in retro-query for indicator {indicator}"
)
if not retro_result or len(retro_result) != 2:
return
reported = 0
stdout = retro_result[0]
for line in stdout.decode().split("\n"):
line = line.rstrip()
if line:
sighting = query_result_to_sighting(line, indicator)
if not sighting:
logger.error(f"Could not parse VAST query result: {line}")
continue
reported += 1
await sightings_queue.put(sighting)
logger.debug(f"Retro-matched {reported} sighting(s) for indicator: {indicator}")
s_retro_matches_per_ioc.observe(reported)
s_retro_query_time_s_per_ioc.observe(time.time() - start)
class TestCallStackCreation(unittest.TestCase):
def setUp(self):
self.vast = VAST(binary="/opt/tenzir/bin/vast")
def test_call_chaining(self):
self.assertEqual(self.vast.call_stack, [])
query = "#timestamp < 1 hour ago"
self.vast.export().arrow(query)
self.assertEqual(self.vast.call_stack, ["export", "arrow", query])
def test_boolean_flag_handling(self):
self.assertEqual(self.vast.call_stack, [])
query = ":addr == 192.168.1.104 && #timestamp < 1 hour ago"
self.vast.export(continuous=True).json(query)
self.assertEqual(
self.vast.call_stack, ["export", "--continuous", "json", query]
)
def test_import_keyword(self):
self.assertEqual(self.vast.call_stack, [])
path = "/some/file"
self.vast.import_().pcap(read=path)
self.assertEqual(self.vast.call_stack, ["import", "pcap", f"--read={path}"])
def test_underscore_replacement_in_parameters(self):
self.assertEqual(self.vast.call_stack, [])
self.vast.export(max_events=10).json("192.168.1.104")
self.assertEqual(
self.vast.call_stack, ["export", "--max-events=10", "json", "192.168.1.104"]
)
def test_underscore_replacement_in_subcommands(self):
self.assertEqual(self.vast.call_stack, [])
self.vast.matcher().ioc_remove(name="foo", ioc="bar")
self.assertEqual(
self.vast.call_stack, ["matcher", "ioc-remove", "--name=foo", "--ioc=bar"]
)
async def continuous_query_example():
"""
This example demonstrates how continuous queries of VAST can be wrapped
with the Python bindings.
To spawn continuous events in VAST, first run this code and then, on a
different terminal, ingest new Zeek logs with VAST. That should update
the console that is running this example code.
"""
vast = VAST(binary="/opt/tenzir/bin/vast")
proc = await vast.export(continuous=True
).json('#type == "zeek.conn"').exec()
print("Waiting for VAST to ingest data...")
for _ in range(10): # print 10 updates
data = await proc.stdout.readline()
print("Ingested new data:")
print(data.decode("ascii").rstrip(), "\n")
async def remove_vast_ioc(vast_binary: str, vast_endpoint: str, indicator: Indicator):
"""
Converts the given STIX-2 Indicator to a VAST-compatible IoC and removes it
from the VAST matcher.
@param vast_binary The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node ('host:port')
@param indicator The STIX-2 Indicator to remove from VAST
"""
global logger, matcher_name
type_and_value = get_vast_type_and_value(indicator.pattern)
if not type_and_value:
logger.debug(f"Cannot remove IoC from VAST. Is it a point IoC? {indicator}")
return None
(vast_type, ioc_value) = type_and_value
vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
# TODO pass matcher_name once VAST supports more fine-grained deletion
await vast.matcher().intel().remove(ioc_value, vast_type).exec()
logger.debug(f"Removed indicator from VAST live matching: {indicator}")
async def example():
print("normal query")
vast = VAST(binary="/opt/tenzir/bin/vast")
await vast.test_connection()
proc = await vast.export(max_events=2
).json(":addr == 192.168.1.104").exec()
stdout, stderr = await proc.communicate(
) # wait until all pipes reached EOF
print(stdout)
print("query with apache arrow export")
proc = await vast.export(max_events=2
).arrow(":addr == 192.168.1.104").exec()
stdout, stderr = await proc.communicate(
) # wait until all pipes reached EOF
reader = pyarrow.ipc.open_stream(stdout)
table = reader.read_all()
print(table.to_pydict())
async def ingest_vast_ioc(vast_binary: str, vast_endpoint: str, indicator: Indicator):
"""
Converts the given STIX-2 Indicator to a VAST-compatible IoC and ingests it
via a VAST matcher.
@param vast_binary The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node ('host:port')
@param indicator The STIX-2 Indicator to query VAST for
"""
global logger
vast_ioc = indicator_to_vast_matcher_ioc(indicator)
if not vast_ioc:
logger.error(
f"Unable to convert STIX-2 Indicator to VAST compatible IoC. Is it a point IoC? {indicator}"
)
return
vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
proc = await vast.import_(type="intel.indicator").json().exec(stdin=vast_ioc)
await proc.wait()
logger.debug(f"Ingested indicator for VAST live matching: {indicator}")
async def receive_intel(cmd, vast_endpoint, pub_endpoint, topic):
"""
Starts a zmq subscriber on the given endpoint and listens new intel
items on the given topic.
@param cmd The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node
@param pub_endpoint A host:port string to connect to via zmq
@param topic The topic to subscribe to get intelligence items
"""
vast = VAST(binary=cmd, endpoint=vast_endpoint)
socket = zmq.Context().socket(zmq.SUB)
socket.connect(f"tcp://{pub_endpoint}")
socket.setsockopt(zmq.SUBSCRIBE, topic.encode())
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
logger.info(f"Receiving intelligence items on {pub_endpoint}/{topic}")
while True:
socks = dict(poller.poll(timeout=10))
if socket in socks and socks[socket] == zmq.POLLIN:
try:
_, msg = socket.recv().decode().split(" ", 1)
intel = json.loads(msg)
except Exception as e:
logger.error(f"Error decoding message: {e}")
continue
operation = intel.get("operation", None)
intel.pop("operation", None)
if operation == "ADD":
proc = (
await vast.import_()
.json(type="intel.indicator")
.exec(stdin=json.dumps(intel))
)
await proc.wait()
logger.debug(f"Ingested intel: {intel}")
elif operation == "REMOVE":
logger.warning("Removal of indicators is not yet supported.")
else:
logger.warning(f"Unsupported operation for indicator: {intel}")
else:
await asyncio.sleep(0.01) # free event loop for other tasks
async def report_sightings(cmd, vast_endpoint, sub_endpoint):
"""
Starts a ZeroMQ publisher on the given endpoint and publishes new sightings
@param cmd The VAST binary command to use with PyVAST
@param vast_endpoint The endpoint of a running VAST node
@param sub_endpoint A host:port string to connect to via ZeroMQ
"""
vast = VAST(binary=cmd, endpoint=vast_endpoint)
socket = zmq.Context().socket(zmq.PUB)
socket.connect(f"tcp://{sub_endpoint}")
topic = "vast/sightings"
logger.info(f"Forwarding sightings to {sub_endpoint}/{topic}")
proc = await vast.export(continuous=True).json('#type == "intel.sighting"').exec()
while True:
data = await proc.stdout.readline()
try:
sighting = data.decode("utf-8").rstrip()
json.loads(sighting) # validate
socket.send_string(f"{topic} {sighting}")
logger.debug(f"Reported sighting: {sighting}")
except Exception as e:
logger.error(f"Cannot parse sighting-output from vast: {data}", e)
async def start(cmd, vast_endpoint, zmq_endpoint, snapshot):
"""
Starts the bridge between the two given endpoints. Subscribes the
configured VAST instance for threat intelligence (IoCs) and reports new
intelligence to Threat Bus.
@param cmd The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node
@param zmq_endpoint The ZMQ management endpoint of Threat Bus
@param snapshot An integer value to request n days of past intel items
"""
vast = VAST(binary=cmd, endpoint=vast_endpoint)
assert await vast.test_connection() is True, "Cannot connect to VAST"
logger.info(f"Calling Threat Bus management endpoint {zmq_endpoint}")
reply = subscribe(zmq_endpoint, "threatbus/intel", snapshot)
if not reply or not isinstance(reply, dict):
logger.error("Subsription unsuccessful")
exit(1)
pub_endpoint = reply.get("pub_endpoint", None)
sub_endpoint = reply.get("sub_endpoint", None)
topic = reply.get("topic", None)
if not pub_endpoint or not sub_endpoint or not topic:
logger.error("Unparsable subscription reply")
exit(1)
logger.info(f"Subscription successfull")
intel_task = asyncio.create_task(
receive_intel(cmd, vast_endpoint, pub_endpoint, topic)
)
sighting_task = asyncio.create_task(
report_sightings(cmd, vast_endpoint, sub_endpoint)
)
atexit.register(unsubscribe, zmq_endpoint, topic)
atexit.register(intel_task.cancel)
atexit.register(sighting_task.cancel)
await asyncio.gather(intel_task, sighting_task)
async def start(
vast_binary: str,
vast_endpoint: str,
zmq_endpoint: str,
snapshot: int,
live_match: bool,
retro_match: bool,
retro_match_max_events: int,
retro_match_timeout: float,
max_open_files: int,
metrics_interval: int,
metrics_filename: str,
transform_cmd: str = None,
sink: str = None,
):
"""
Starts the app between the two given endpoints. Subscribes the configured
VAST instance for threat intelligence (IoCs) and reports new sightings to
Threat Bus.
@param cmd The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running VAST node ('host:port')
@param zmq_endpoint The ZMQ management endpoint of Threat Bus ('host:port')
@param snapshot An integer value to request n days of historical IoC items
@param live_match Boolean flag to enable live-matching
@param retro_match Boolean flag to enable retro-matching
@param retro_match_max_events Max amount of retro match results
@param retro_match_timeout Interval after which to terminate the retro-query
@param max_open_files The maximum number of concurrent background tasks for VAST queries.
@param merics_interval The interval in seconds to bucketize metrics
@param metrics_filename The filename (system path) to store metrics at
@param transform_cmd The command to use to transform Sighting context with
@param sink Forward sighting context to this sink (subprocess) instead of
reporting back to Threat Bus
"""
global logger, async_tasks, p2p_topic, max_open_tasks, metrics
# needs to be created inside the same eventloop where it is used
max_open_tasks = asyncio.Semaphore(max_open_files)
vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
assert await vast.test_connection() is True, "Cannot connect to VAST"
logger.debug(f"Calling Threat Bus management endpoint {zmq_endpoint}")
reply = subscribe(zmq_endpoint, "stix2/indicator", snapshot)
if not reply_is_success(reply):
logger.error("Subscription failed")
return
pub_endpoint = reply.get("pub_endpoint", None)
sub_endpoint = reply.get("sub_endpoint", None)
topic = reply.get("topic", None)
if not pub_endpoint or not sub_endpoint or not topic:
logger.error("Subscription failed")
return
logger.info(f"Subscription successful. New p2p_topic: {topic}")
if p2p_topic:
# The 'start' function is called as result of a restart
# Unsubscribe the old topic as soon as we get a working connection
logger.info("Cleaning up old p2p_topic subscription ...")
unsubscribe(zmq_endpoint, p2p_topic)
atexit.unregister(unsubscribe)
p2p_topic = topic
atexit.register(unsubscribe, zmq_endpoint, topic)
async_tasks.append(
asyncio.create_task(heartbeat(zmq_endpoint, p2p_topic, interval=5))
)
indicator_queue = asyncio.Queue()
sightings_queue = asyncio.Queue()
async_tasks.append(
asyncio.create_task(
report_sightings(sub_endpoint, sightings_queue, transform_cmd, sink)
)
)
async_tasks.append(
asyncio.create_task(receive(pub_endpoint, p2p_topic, indicator_queue))
)
async_tasks.append(
asyncio.create_task(
match_intel(
vast_binary,
vast_endpoint,
indicator_queue,
sightings_queue,
live_match,
retro_match,
retro_match_max_events,
retro_match_timeout,
)
)
)
if retro_match:
# add metrics for retro-matching to the metric output
metrics += [s_retro_matches_per_ioc, s_retro_query_time_s_per_ioc]
if live_match:
# add metrics for live-matching to the metric output
metrics.append(g_live_matcher_sightings)
async_tasks.append(
asyncio.create_task(
live_match_vast(vast_binary, vast_endpoint, sightings_queue)
)
)
if metrics_interval:
async_tasks.append(
asyncio.create_task(write_metrics(metrics_interval, metrics_filename))
)
loop = asyncio.get_event_loop()
for s in [signal.SIGHUP, signal.SIGTERM, signal.SIGINT]:
loop.add_signal_handler(s, lambda: asyncio.create_task(stop_signal()))
return await asyncio.gather(*async_tasks)
def setUp(self):
self.vast = VAST(binary="/opt/tenzir/bin/vast") # default port
self.vast_disconnected = VAST(
binary="/opt/tenzir/bin/vast",
endpoint="localhost:44883") # closed / unbound port
def setUp(self):
self.vast = VAST(binary="/opt/tenzir/bin/vast")
