def main():
'''Entry point'''
ndr_config = ndr.Config('/etc/ndr/config.yml')
status_message = ndr.StatusMessage(ndr_config)
status_message.sign_report()
status_message.load_into_queue()
def main():
# Get our list of log files
parser = argparse.ArgumentParser(
description="Converts a TCPDump PCAP file to a traffic report message and sends it")
parser.add_argument('-c', '--config',
default='/etc/ndr/config.yml',
help='NDR Configuration File')
parser.add_argument('-k', '--keep', help='keep pcap file', action='store_true')
parser.add_argument('pcaps', nargs='+',
help='Packet Capture Files')
args = parser.parse_args()
ndr_config = ndr.Config(args.config)
logger = ndr_config.logger
for pcap in args.pcaps:
trm = ndr.TrafficReportMessage(ndr_config)
logger.info("Processing %s", pcap)
trm.parse_pcap_file(pcap)
logger.debug("Processed %d entries", len(trm.traffic_entries))
logger.debug("Skipped %d local LAN entries", trm.filtered_entries)
trm.sign_report()
trm.load_into_queue()
# Because TCPDump doesn't clean up after itself after processing
if args.keep is not True:
os.remove(pcap)
def main():
'''Starts the scan network'''
# We need the NDR Network config for this scan
ndr_config = ndr.Config('/etc/ndr/config.yml')
parser = argparse.ArgumentParser(
description="Intelligently scans the network with NMAP")
parser.add_argument('--net-config',
default=ndr_config.ndr_netconfig_file,
help='Network Configuration File')
parser.add_argument('--nmap-config',
default=ndr_config.nmap_configuration_file,
help='NMAP Configuration File')
# Load in the NDR configuration
args = parser.parse_args()
if os.getuid() != 0:
print("ERROR: must be run as root")
return
nmap_config = ndr.NmapConfig(args.net_config)
nmap_runner = ndr.NmapRunner(ndr_config, nmap_config)
nmap_runner.run_network_scans()
def setUpClass(cls):
'''Sets everything up for testing status messages'''
cls._current_time = int(time.time())
cls._ncc = ndr.Config(NDR_CONFIG)
# Create a temporary file for the OTA timestamp
file_descriptor, cls._ota_timestamp = tempfile.mkstemp()
with os.fdopen(file_descriptor, 'w') as ts_file:
ts_file.write(str(cls._current_time))
cls._ncc.image_timestamp_file = cls._ota_timestamp
def main():
parser = argparse.ArgumentParser(
description=
"Uploads SNORT all-traffic CSV files. If directory is specified, all files in that directory"
)
parser.add_argument('-f',
"--filename",
default='log.csv',
help='base name of the log file to upload')
parser.add_argument('logs', nargs='+', help='log files to upload')
args = parser.parse_args()
ndr_config = ndr.Config('/etc/ndr/config.yml')
snort_traffic_log = ndr.SnortTrafficLog(ndr_config)
logger = ndr_config.logger
for logpath in args.logs:
if os.path.isfile(logpath):
logger.debug("Processing log: %s", logpath)
snort_traffic_log.append_log(logpath)
continue
logger.debug("Processing log directory: %s", logpath)
for _, _, filelist in os.walk(logpath):
filelist.sort() # Causes the files to be spat out oldest first
# We're not interested in subdirectories so
for filename in filelist:
logfile = logpath + "/" + filename
try:
if filename == args.filename:
logger.debug("Skipping base log file: %s", filename)
continue
logger.debug("Parsing: %s", filename)
snort_traffic_log.append_log(logfile)
snort_traffic_log.consolate()
# Delete the log file after we're done with it
os.remove(logfile)
except:
trace = traceback.format_exc()
logger.error("log parse died with error: %s", trace)
# Can't do this in a finally block as we need to skip the base log file
# or SNORT ABENDs
os.remove(logfile)
snort_traffic_log.sign_report()
snort_traffic_log.load_into_queue()
def main():
# Get our list of log files
parser = argparse.ArgumentParser(
description="Upload a JSON-formatted syslog file for processing")
parser.add_argument('-c', '--config',
default='/etc/ndr/config.yml',
help='NDR Configuration File')
parser.add_argument('logs', nargs='+',
help='log files to upload')
args = parser.parse_args()
ndr_config = ndr.Config(args.config)
logger = ndr_config.logger
# This is slagging ugly. Nothing stops us from getting
# a bad file in so we need to handle that case, and then
# pop a message off for each log
# YAML is a superset of JSON so we should be able to load
# each line of the log file and pull the object out of with
# Because of the way syslog-ng logs JSON, we can't simply
# grab the entire thing because there isn't array markers in
# the file ..,
for log in args.logs:
log_upload = SyslogUploadMessage(ndr_config)
try:
with open(log, 'r') as f:
for line in f:
# It's also possible we can get a bad entry. In that case, skip it and report
# it into the log. It will get dumped into syslog_upload's syslog error log
# for autospy
try:
yaml_line = yaml.safe_load(line)
entry = SyslogEntry.from_dict(yaml_line)
if entry == None: # Was a newline, discarding
continue
log_upload.add_entry(entry)
except ValueError:
logger.error("failed to parse %s in %s", line, log)
except:
logger.error("cataphoric error %s %s %s", log, line, sys.exc_info()[0])
# With everything loaded, queue the magic
log_upload.sign_report()
log_upload.load_into_queue()
finally:
pass
def main():
parser = argparse.ArgumentParser(
description="Upload a status report to the server")
parser.add_argument('-c', '--config',
default='/etc/ndr/config.yml',
help='NDR Configuration File')
args = parser.parse_args()
ndr_config = ndr.Config(args.config)
ingest_message = ndr.IngestMessage(
ndr_config, ndr.IngestMessageTypes.TEST_ALERT
)
ingest_message.sign_report()
ingest_message.load_into_queue()
def main():
'''Entry point'''
parser = argparse.ArgumentParser(
description="Upload a status report to the server")
parser.add_argument('-c',
'--config',
default='/etc/ndr/config.yml',
help='NDR Configuration File')
args = parser.parse_args()
ndr_config = ndr.Config(args.config)
status_message = ndr.StatusMessage(ndr_config)
status_message.sign_report()
status_message.load_into_queue()
def setup_ndr_client_config(self):
logging.getLogger().addHandler(logging.NullHandler())
self._ncc = ndr.Config(NDR_CONFIG_FILE)
self._ncc.logger = logging.getLogger()
self._ncc.image_information_file = IMAGE_CONFIG
self._created_files = []
config_ndr_for_signing_and_local_queue(self)
# Create a temporary directory for handling config files that are optional but may get
# written out or updated or something.
self._ncc_config_dir = tempfile.mkdtemp()
# Override optional config files with paths that won't conflict
self._ncc.nmap_configuration_file = self._ncc_config_dir + "/nmap_config.yml"
# Write out the test config for testing mainrun programs
self._ndr_config_file = create_temp_file(self)
with open(self._ndr_config_file, 'w') as f:
yaml_content = yaml.dump(self._ncc.to_dict())
f.write(yaml_content)
def main():
# Do our basic setup work
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
logger = logging.getLogger(name=__name__)
logger.setLevel(logging.DEBUG)
# We need both configs
ncc = ndr.Config("/etc/ndr/config.yml") # NDR Client Config
nsc = ndr_server.Config(logger, "/etc/ndr/ndr_server.yml")
db_connection = nsc.database.get_connection()
parser = argparse.ArgumentParser(
description="Requests that a recorder restart to install OTAs updates")
parser.add_argument('recorders', nargs='+', help='recorders to reboot')
args = parser.parse_args()
for recorder in args.recorders:
# Make sure the recorder exists
try:
# We'll try to initialize a Recorder object. We don't need it but it confirms
# that the recorder exists in the datbase
ndr_server.Recorder.read_by_hostname(nsc,
recorder,
db_conn=db_connection)
msg = ndr.IngestMessage(
config=ncc, message_type=ndr.IngestMessageTypes.REBOOT_REQUEST)
msg.destination = recorder
msg.upload_method = 'uux'
msg.sign_report()
msg.load_into_queue()
logger.info("Queued recorder %s to reboot", recorder)
except psycopg2.DatabaseError:
logger.error("recorder %s does not exist", recorder)
db_connection.close()
def main():
# Get our list of log files
parser = argparse.ArgumentParser(
description="Process a remote message")
parser.add_argument('-k', "--keep", help='base name of the log file to upload', action='store_true')
parser.add_argument('messages', nargs='+',
help='remote messages to process')
parser.add_argument('-c', '--config',
default='/etc/ndr/config.yml',
help='NDR Configuration File')
args = parser.parse_args()
ndr_config = ndr.Config(args.config)
logger = ndr_config.logger
validated_messages = []
for message_file in args.messages:
# DEBUG, write message to file
if not os.path.isdir("/tmp/incoming_messages"):
os.makedirs("/tmp/incoming_messages")
logger.info("Processing %s", message_file)
shutil.copy(
message_file, "/tmp/incoming_messages/" + os.path.basename(message_file))
# If we're operating in UUCP mode, then we only accept messages signed
# by our ingest
accepted_cns = None
if ndr_config.upload_method == 'uucp':
accepted_cns = ndr_config.ingest_uucp_host
else:
logger.warning("accepting any signed messages due to local mode!")
message = ndr.IngestMessage.verify_and_load_message(
ndr_config, message_file, only_accept_cn=accepted_cns)
if message is not None:
logger.info("Successfully parsed and validated the message!")
# Now some magic is required. If we're running via uux, we're running in the context of
# the uucp user and not under NDR. The UUCP user can run ndr-process-message as sudo so
# we need to copy the message over to a temporary directory if we're not root, and then
# recurse into ourselves. We'll do this for all the messages in the stack.
validated_messages.append(message_file)
# If we're root, just process it
if os.getuid() == 0:
logger.info("Running in root context")
# Now do things with it based on the type of message it is
if message.message_type == ndr.IngestMessageTypes.CERTIFICATE_REQUEST:
logger.info("Got a certificate request message ...")
cert_request_msg = ndr.CertificateRequest()
cert_request_msg.from_message(message)
# Write out the signed certificates to the root filesystem
if cert_request_msg.certificate is not None:
logger.info("Writing out signed certificate ...")
with open(ndr_config.ssl_certfile, 'w') as f:
f.write(cert_request_msg.certificate)
if cert_request_msg.certificate_chain is not None:
logger.info("Writing out certificate chain ...")
with open(ndr_config.ssl_bundle, 'w') as f:
f.write(cert_request_msg.certificate_chain)
logger.info("Updated certificate chain for device")
elif message.message_type == ndr.IngestMessageTypes.REBOOT_REQUEST:
logger.info("Got a reboot request")
shutdown_args = [ "shutdown", "-r", "now"]
shutdown_process = subprocess.run(
args=shutdown_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
check=False)
if shutdown_process.returncode != 0:
logger.error("shutdown failed: %s", str(shutdown_process.stderr, 'utf-8'))
return
elif message.message_type == ndr.IngestMessageTypes.FILE_UPDATE:
logger.info("Got a file update message")
file_update_message = ndr.FileUpdateMessage(ndr_config)
file_update_message.from_message(message)
file_update_message.write_updates()
else:
logger.error("Got non-client accepted %s message", message.message_type.value)
else:
logger.info("Not running as root, saving messages to run in sudo")
# Non-root fall through. We should have a pile of validated messages
if os.getuid() != 0:
files_for_sudo = []
for valid_message in validated_messages:
# The messages will be revalidated on the second go around
msg_fd, vm_file = tempfile.mkstemp()
os.close(msg_fd)
shutil.copy(valid_message, vm_file)
files_for_sudo.append(vm_file)
# Attempt to call ourselves via sudo
sudo_process_message = [ "sudo", "ndr-process-message"]
sudo_process_message += files_for_sudo
# Here goes nothing
sudo_proc = subprocess.run(
args=sudo_process_message, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
check=False)
if sudo_proc.returncode != 0:
logger.error("sudo run failed: %s", str(sudo_proc.stderr, 'utf-8'))
return
# And done (we log this here so it only prints once)
logger.info("Finished processing messages")
# Make a final loop through to clean up our files
if args.keep is False:
for message in args.messages:
os.remove(message)
return
import ndr
# Testing data from a live system running syslog-ng in JSON reporting mode
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_SURFACE_SCAN_DATA = THIS_DIR + '/data/nmap_v4_surface.xml'
TEST_V6_LINK_LOCAL_SCAN_DATA = THIS_DIR + '/data/nmap_v6_link_local.xml'
TEST_HOSTNAME_SCAN_DATA = THIS_DIR + '/data/nmap_hostname_test.xml'
TEST_SD_SCAN_DATA = THIS_DIR + '/data/nmap_service_discovery.xml'
TEST_MULTIHOME_DATA = THIS_DIR + '/data/nmap_multihome.xml'
TEST_MULTIHOME_V6_DATA = THIS_DIR + '/data/nmap_multihome_v6.xml'
TEST_LINK_LOCAL_PARSING = THIS_DIR + '/data/nmap_ipv6_link_local_scan.yml'
OUTPUT_PRETTY_PRINT_SD_SCAN = THIS_DIR + '/data/expected_outputs/pretty_print_sd_scan'
NDR_CONFIG = ndr.Config(THIS_DIR + '/data/test_config.yml')
def load_nmap_xml_data(file_name):
'''Helper function to load XML data'''
with open(file_name, 'r') as xml_file:
return xml_file.read()
class NmapTest(unittest.TestCase):
'''Tests functionality related to NMAP scanning and parsing'''
maxDiff = None
def test_scandata_parsing(self):
'''Tests parsing of the basic surface scan'''
def main():
config = ndr.Config('/etc/ndr/config.yml')
# Unlike the majority of NDR tools, this one is meant to run interactivity
print("Network Data Recorder Enlistment Tool")
print("Copyright (C) 2017 - Secured By THEM")
if os.getuid() != 0:
print("ERROR: This utility must be run as root!")
sys.exit(-1)
hostname = config.hostname
print("Hostname: ", hostname)
# Print out some helpful messages, then ask questions
print('''
This script will automatically generate a certificate signing request
and other security keys for this recorder. The information used in the CSR
will be used to create an organization and site if necessary, and the pseudonym
will become the default human readable name used to refer to this recorder.
Manual processing of the CSR is required server side. After uploading the CSR,
we'll poll to wait for the CSR to be signed and downloaded. If you don't wish to
enlist this recorder now, press Ctrl-C
NOTE: Names are case sensitive!
''')
got_info = False
while got_info is False:
organization = ""
while organization == "":
organization = input("Organization: ")
if organization == "":
print("Organization can not be blank!")
org_unit = ""
while org_unit == "":
org_unit = input("Organization Unit/Site: ")
if organization == "":
org_unit("OU can not be blank!")
pseudonym = ""
while pseudonym == "":
pseudonym = input("Pseudonym/Human Name: ")
if organization == "":
pseudonym("Pseudonym can not be blank!")
print()
print("=== CSR to be generated ===")
print("Organization:", organization)
print("Organization Unit (Site):", org_unit)
print("Pseudonym (Human Name):", pseudonym)
print("Common Name:", hostname)
print()
confirmation = input("Is this information correct? [N/y] ")
if confirmation and confirmation.lower()[0] == 'y':
got_info = True
# Check if we have a private key
print("STEP 1: Checking cryptography keys ...")
key = None
if os.path.isfile(config.ssl_private_key) is False:
print("No private key found, generating. This may take some time ...")
key = rsa.generate_private_key(public_exponent=65537,
key_size=2048,
backend=default_backend())
with open(config.ssl_private_key, "wb") as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()))
print("Generated ", config.ssl_private_key)
else:
print("Have recorder private key")
with open(config.ssl_private_key, "rb") as f:
key = load_pem_private_key(f.read(),
password=None,
backend=default_backend())
# Now generate a certificate signing request (we can always generate a new
# one safely)
print("STEP 2: Generating CSR ...")
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, hostname),
x509.NameAttribute(NameOID.ORGANIZATION_NAME, organization),
x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, org_unit),
x509.NameAttribute(NameOID.PSEUDONYM, pseudonym)
])).sign(key, hashes.SHA256(), default_backend())
with open(config.ssl_csr, "wb") as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
print("CSR written to ", config.ssl_csr)
# Uploading the report is a bit different since we don't have a local certificate to sign it
# with. First, let's create a CertificateRequest message
csr_msg = ndr.CertificateRequest(config)
csr_msg.csr = str(csr.public_bytes(serialization.Encoding.PEM), 'utf-8')
csr_msg.destination_queue = ndr.IngestMessageDestinations.ENROLLMENT_QUEUE
# Now we create the report, and load it into signed message and send it on
# its way
csr_msg.create_report()
csr_msg.signed_message = csr_msg.message
csr_msg.load_into_queue()