def dumpling_emitter(
kitchen_name: str,
hub: str,
dumpling_queue: multiprocessing.Queue,
kitchen_info: Dict,
):
"""
Starts an async event loop to manage funneling dumplings from the queue to
the dumpling hub.
This function is intended to be invoked as a Python process.
:param kitchen_name: The name of the kitchen that the dumplings will be
coming from.
:param hub: The address where ``nd-hub`` is receiving dumplings.
:param dumpling_queue: Queue to get dumplings from.
:param kitchen_info: Information on the kitchen.
"""
configure_logging()
log = logging.getLogger('netdumplings.console.sniff')
log.info("{0}: Starting dumpling emitter process".format(kitchen_name))
try:
asyncio.run(
send_dumplings_from_queue_to_hub(kitchen_name, hub, dumpling_queue,
kitchen_info, log))
except KeyboardInterrupt:
log.warning(f"Keyboard interrupt detected")
def test_logging_formatter(self):
"""
Test that the logging formatter gets configured like we expect.
"""
configure_logging()
assert logging.Formatter.converter == time.gmtime
assert logging.Formatter.default_time_format == '%Y-%m-%dT%H:%M:%S'
assert logging.Formatter.default_msec_format == '%s.%03d'
def test_invalid_logging_config_file(self, monkeypatch, mocker):
"""
Test NETDUMPLINGS_LOGGING_CONFIG environment variable for setting the
logging config with an invalid file. Should fall back on basicConfig().
"""
# We still want the Formatter to be configured.
assert logging.Formatter.converter == time.gmtime
assert logging.Formatter.default_time_format == '%Y-%m-%dT%H:%M:%S'
assert logging.Formatter.default_msec_format == '%s.%03d'
# Set NETDUMPLINGS_LOGGING_CONFIG to point to a test logging config.
logging_config_file = 'tests/data/logging_bogus.json'
monkeypatch.setenv('NETDUMPLINGS_LOGGING_CONFIG', logging_config_file)
spy_basic_config = mocker.spy(logging, 'basicConfig')
configure_logging(logging.DEBUG)
spy_basic_config.assert_called_once_with(level=logging.DEBUG)
def hub_cli(address, in_port, out_port, status_freq):
"""
The dumpling hub.
Sends dumplings received from all kitchens (usually any running instances
of nd-sniff) to all dumpling eaters. All kitchens and eaters need to
connect to the nd-hub --in-port or --out-port respectively.
"""
configure_logging()
dumpling_hub = netdumplings.DumplingHub(
address=address,
in_port=in_port,
out_port=out_port,
status_freq=status_freq,
)
try:
dumpling_hub.run()
except NetDumplingsError as e:
print('hub error: {0}'.format(e))
sys.exit(1)
def test_logging_config_file(self, monkeypatch):
"""
Test NETDUMPLINGS_LOGGING_CONFIG environment variable for setting the
logging config.
"""
# We still want the Formatter to be configured.
assert logging.Formatter.converter == time.gmtime
assert logging.Formatter.default_time_format == '%Y-%m-%dT%H:%M:%S'
assert logging.Formatter.default_msec_format == '%s.%03d'
# Set NETDUMPLINGS_LOGGING_CONFIG to point to a test logging config.
logging_config_file = 'tests/data/logging.json'
monkeypatch.setenv('NETDUMPLINGS_LOGGING_CONFIG', logging_config_file)
configure_logging()
# The test config file sets all the loggers to ERROR.
assert logging.getLogger('netdumplings').level == logging.ERROR
assert logging.getLogger(
'netdumplings.dumplinghub').level == logging.ERROR
assert logging.getLogger(
'netdumplings.dumplingkitchen').level == logging.ERROR
assert logging.getLogger(
'netdumplings.dumplingeater').level == logging.ERROR
def sniff_cli(kitchen_name, hub, interface, pkt_filter, chef_module, chef,
chef_list, poke_interval):
"""
A dumpling sniffer kitchen.
Sniffs network packets matching the given PCAP-style filter and sends them
to chefs for processing into dumplings. Dumplings are then sent to nd-hub
for distribution to the dumpling eaters.
"""
# NOTE: Since the --chef-module and --chef flags can be specified multiple
# times, the associated 'chef_module' and 'chef' parameters are tuples of
# zero or more modules/chefs respectively.
chef = True if len(chef) == 0 else chef
# Display the chef list and exit if that's all the user wanted.
if chef_list:
list_chefs(chef_module)
sys.exit(0)
# now do the following:
#
# * Create a queue for a network-sniffing kitchen process to put dumplings
# onto
# * Start a kitchen process, which will be putting dumplings onto the queue
# * Start a dumpling emitter process which takes dumplings from the queue
# and sends them to nd-hub over a websocket
configure_logging()
logger = logging.getLogger('netdumplings.console.sniff')
# A queue for passing dumplings from the sniffer kitchen to the
# dumpling-emitter process.
dumpling_emitter_queue = multiprocessing.Queue()
# Determine what chefs we'll be sending packets to.
valid_chefs = get_valid_chefs(kitchen_name, chef_module, chef, logger)
if not valid_chefs:
logger.error('{}: No valid chefs found. Not starting sniffer.'.format(
kitchen_name
))
sys.exit(1)
# Generate list of module.class names for all the seemingly-valid chefs
# we'll be instantiating. This is for use in the status dumplings.
valid_chef_list = []
for chef_module_name in sorted(valid_chefs.keys()):
for chef_class_name in sorted(valid_chefs[chef_module_name]):
valid_chef_list.append('{}.{}'.format(
chef_module_name, chef_class_name)
)
# Start the sniffer kitchen and dumpling-emitter processes.
sniffer_process = multiprocessing.Process(
target=network_sniffer,
args=(
kitchen_name, interface, chef, chef_module, valid_chefs,
pkt_filter, poke_interval, dumpling_emitter_queue,
)
)
kitchen_info = {
'kitchen_name': kitchen_name,
'interface': interface,
'filter': pkt_filter,
'chefs': valid_chef_list,
'poke_interval': poke_interval,
}
dumpling_emitter_process = multiprocessing.Process(
target=dumpling_emitter,
args=(kitchen_name, hub, dumpling_emitter_queue, kitchen_info),
)
sniffer_process.start()
dumpling_emitter_process.start()
try:
while True:
if (sniffer_process.is_alive() and
dumpling_emitter_process.is_alive()):
sleep(1)
else:
if sniffer_process.is_alive():
logger.error(
"{0}: Dumpling emitter process died; exiting.".format(
kitchen_name))
sniffer_process.terminate()
if dumpling_emitter_process.is_alive():
logger.error(
"{0}: Network sniffer process died; exiting.".format(
kitchen_name))
dumpling_emitter_process.terminate()
break
except KeyboardInterrupt:
logger.warning(
"{0}: Caught keyboard interrupt; exiting.".format(
kitchen_name))
def sniff_cli(kitchen_name, hub, interface, pkt_filter, chef_module, chef,
chef_list, poke_interval):
"""
A dumpling sniffer kitchen.
Sniffs network packets matching the given PCAP-style filter and sends them
to chefs for processing into dumplings. Dumplings are then sent to nd-hub
for distribution to the dumpling eaters.
This tool likely needs to be run as root, or as an Administrator user.
"""
# NOTE: Since the --chef-module and --chef flags can be specified multiple
# times, the associated 'chef_module' and 'chef' parameters are tuples of
# zero or more modules/chefs respectively.
chef = True if len(chef) == 0 else chef
# Display the chef list and exit if that's all the user wanted.
if chef_list:
list_chefs(chef_module)
sys.exit(0)
# now do the following:
#
# * Create a queue for a network-sniffing kitchen process to put dumplings
# onto
# * Start a kitchen process, which will be putting dumplings onto the queue
# * Start a dumpling emitter process which takes dumplings from the queue
# and sends them to nd-hub over a websocket
configure_logging()
logger = logging.getLogger('netdumplings.console.sniff')
logger.info("Initializing sniffer...")
# A queue for passing dumplings from the sniffer kitchen to the
# dumpling-emitter process.
dumpling_emitter_queue = multiprocessing.Queue()
# Determine what chefs we'll be sending packets to.
valid_chefs = get_valid_chefs(kitchen_name, chef_module, chef, logger)
if not valid_chefs:
logger.error('{}: No valid chefs found. Not starting sniffer.'.format(
kitchen_name))
sys.exit(1)
# Generate list of module.class names for all the seemingly-valid chefs
# we'll be instantiating. This is for use in the status dumplings.
valid_chef_list = []
for chef_module_name in sorted(valid_chefs.keys()):
for chef_class_name in sorted(valid_chefs[chef_module_name]):
valid_chef_list.append('{}.{}'.format(chef_module_name,
chef_class_name))
# Start the sniffer kitchen and dumpling-emitter processes.
sniffer_process = multiprocessing.Process(target=network_sniffer,
args=(
kitchen_name,
interface,
chef,
chef_module,
valid_chefs,
pkt_filter,
poke_interval,
dumpling_emitter_queue,
))
kitchen_info = {
'kitchen_name': kitchen_name,
'interface': interface,
'filter': pkt_filter,
'chefs': valid_chef_list,
'poke_interval': poke_interval,
}
dumpling_emitter_process = multiprocessing.Process(
target=dumpling_emitter,
args=(kitchen_name, hub, dumpling_emitter_queue, kitchen_info),
daemon=True,
)
sniffer_process.start()
dumpling_emitter_process.start()
try:
while True:
sniffer_process.join(0.5)
dumpling_emitter_process.join(0.5)
if not sniffer_process.is_alive():
logger.error(
f"{kitchen_name}: Network sniffer process died; exiting.")
break
if not dumpling_emitter_process.is_alive():
logger.error(
f"{kitchen_name}: Dumpling emitter process died; exiting.")
break
except KeyboardInterrupt:
logger.warning(f"{kitchen_name}: Caught keyboard interrupt; exiting.")
for process in multiprocessing.active_children():
process.terminate()
def network_sniffer(
kitchen_name: str,
interface: str,
chefs: Union[List[str], bool],
chef_modules: List[str],
valid_chefs: Dict,
sniffer_filter: str,
chef_poke_interval: int,
dumpling_queue: multiprocessing.Queue,
):
"""
The main network sniffing management function, responsible for:
* Instantiating a dumpling kitchen (which does the actual sniffing) and
providing it with a queue to put chef-created dumplings on
* Instantiating the dumpling chefs and registering them with the kitchen
* Running the kitchen
This function is intended to be invoked as a Python process.
A dumpling chef will only be instantiated if its ``assignable_to_kitchen``
class attribute is ``True``.
:param kitchen_name: Name of the sniffer kitchen.
:param interface: Network interface to sniff (``all`` sniffs all
interfaces).
:param chefs: List of chefs to send packets to. Used for display only.
:param chef_modules: List of Python module names in which to find chefs.
Used for display only.
:param valid_chefs: Dict of module+chef combinations we plan on importing.
:param sniffer_filter: PCAP-compliant sniffer filter.
:param chef_poke_interval: Interval (in secs) to poke chefs.
:param dumpling_queue: Queue to pass to the kitchen to put dumplings on.
"""
configure_logging()
log = logging.getLogger('netdumplings.console.sniff')
log.info("{0}: Starting network sniffer process".format(kitchen_name))
log.info("{0}: Interface: {1}".format(kitchen_name, interface))
log.info("{0}: Requested chefs: {1}".format(
kitchen_name, "all" if chefs is True else ", ".join(chefs)))
log.info("{0}: Chef modules: {1}".format(kitchen_name,
", ".join(chef_modules)))
log.info("{0}: Filter: {1}".format(
kitchen_name,
"<all packets>" if not sniffer_filter else sniffer_filter))
log.info("{0}: Chef poke interval (secs): {1}".format(
kitchen_name, chef_poke_interval))
sniffer_kitchen = netdumplings.DumplingKitchen(
dumpling_queue=dumpling_queue,
name=kitchen_name,
interface=interface,
sniffer_filter=sniffer_filter,
chef_poke_interval=chef_poke_interval,
)
# Instantiate all the valid DumplingChef classes and register them with
# the kitchen.
for chef_module in valid_chefs:
chef_class_names = valid_chefs[chef_module]
if os.path.isfile(chef_module):
spec = importlib.util.spec_from_file_location('chefs', chef_module)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
else:
# TODO: Investigate replacing __import__ with
# importlib.import_module
mod = __import__(chef_module, fromlist=chef_class_names)
for chef_class_name in chef_class_names:
log.info("{0}: Registering {1}.{2} with kitchen".format(
kitchen_name, chef_module, chef_class_name))
klass = getattr(mod, chef_class_name)
klass(kitchen=sniffer_kitchen)
sniffer_kitchen.run()