def setupDisks(numDisks): print "Setup Disks" devLetters = list(string.ascii_lowercase) for x in range(1, int(numDisks) + 1): try: mkfs("/dev/xvd" + devLetters[x], "-text4", "-E lazy_itable_init=1") except Exception as e: pass if not os.path.exists("/mnt/data" + str(x)): os.makedirs("/mnt/data" + str(x)) try: mount("/dev/xvd" + devLetters[x], "/mnt/data" + str(x), "-text4") except Exception as e: pass try: Fstab.add("/dev/xvd" + devLetters[x], "/mnt/data" + str(x), filesystem="ext4") except Exception as e: pass
def fstab_add(dev, mp, fs, options=None): """Adds the given device entry to the /etc/fstab file """ return Fstab.add(dev, mp, fs, options=options)
def main(): # to parse fdisk output, look for partitions after partitions header line fdisk_pars_begin_pattern = re.compile( r'^Device\s+Start\s+End\s+Sectors\s+Size\s+Type\s*$') # to parse partitions from fdisk output after parted creates partition table fdisk_par_pattern = re.compile( r'^(?P<device>\S+)\s+(?P<start>\d+)\s+(?P<end>\d+)\s+(?P<sectors>\d+)\s+(?P<size>\S+)\s+(?P<type>.*)$' ) # extract arguments from the command line parser = argparse.ArgumentParser( description='sensor-capture-disk-config.py', add_help=False, usage='sensor-capture-disk-config.py [options]') parser.add_argument('-i', '--interactive', dest='interactive', type=str2bool, nargs='?', const=True, default=False, help="Interactive") parser.add_argument( '-u', '--umount', dest='umount', type=str2bool, nargs='?', const=True, default=False, help="Unmount capture directories before determining candidate drives") parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=False, help="Verbose output") parser.add_argument('-n', '--dry-run', dest='dryrun', type=str2bool, nargs='?', const=True, default=False, help="Dry run (don't perform actions)") parser.add_argument('-c', '--crypto', dest='encrypt', type=str2bool, nargs='?', const=True, default=False, help="Encrypt formatted volumes") try: parser.error = parser.exit args = parser.parse_args() except SystemExit: parser.print_help() exit(2) debug = args.debug if debug: eprint(f"Arguments: {sys.argv[1:]}") if debug: eprint(f"Arguments: {args}") # unmount existing mounts if requested if args.umount and (not args.dryrun): if (not args.interactive ) or YesOrNo(f'Unmount any mounted capture path(s)?'): if debug: eprint(f"Attempting unmount of capture path(s)...") run_process( f"umount {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_PCAP_DIR)}" ) run_process( f"umount {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_ZEEK_DIR)}" ) run_process(f"umount {CAPTURE_MOUNT_ROOT_PATH}") # also luksClose any luks volumes devices we might have set up for cryptDev in [ remove_prefix(x, '/dev/mapper/') for x in glob.glob( f"/dev/mapper/{CAPTURE_CRYPT_DEV_PREFIX}*") ]: if debug: eprint(f"Running crypsetup luksClose on {cryptDev}...") _, cryptOut = run_process( f"/sbin/cryptsetup --verbose luksClose {cryptDev}", stdout=True, stderr=True, timeout=300) if debug: for line in cryptOut: eprint(f"\t{line}") _, reloadOut = run_process(f"systemctl daemon-reload") # check existing mounts, if the capture path(s) are already mounted, then abort with open('/proc/mounts', 'r') as f: for line in f.readlines(): mountDetails = line.split() if (len(mountDetails) >= 2): mountPoint = mountDetails[1] if mountPoint.startswith(CAPTURE_MOUNT_ROOT_PATH): eprint( f"It appears there is already a device mounted under {CAPTURE_MOUNT_ROOT_PATH} at {mountPoint}." ) eprint( f"If you wish to continue, you may run this script with the '-u|--umount' option to umount first." ) eprint() parser.print_help() exit(2) # get physical disks, partitions, device maps, and any mountpoints and UUID associated allDisks = defaultdict(list) if debug: eprint(f"Block devices:") for device in GetInternalDevices(): ecode, deviceTree = run_process( f'/bin/lsblk -o name,uuid,mountpoint --paths --noheadings /dev/{device}', stdout=True, stderr=False) if (ecode == 0): currentDev = None currentPar = None currentMapper = None for line in deviceTree: line = line.rstrip() if (len(line) > 0): if debug: eprint(f"\t{line}") if (line == f"/dev/{device}"): currentDev = line currentPar = None currentMapper = None allDisks[currentDev].append( PartitionInfo(device=currentDev)) elif (currentDev is not None) and (line[2:2 + len(f"/dev/{device}")] == f"/dev/{device}"): parInfo = f"/{line.split('/', 1)[-1]}".split() if (len(parInfo) >= 2): currentPar = PartitionInfo( device=currentDev, partition=parInfo[0], uuid=parInfo[1], mount=parInfo[2] if (len(parInfo) > 2) else None) currentMapper = None allDisks[currentDev].append(currentPar) elif (currentPar is not None) and ("/dev/mapper/" in line): parInfo = f"/{line.split('/', 1)[-1]}".split() if (len(parInfo) >= 2): currentMapper = PartitionInfo( device=currentDev, partition=currentPar.partition, mapper=parInfo[0], uuid=parInfo[1], mount=parInfo[2] if (len(parInfo) > 2) else None) allDisks[currentDev].append(currentMapper) # at this point allDisks might look like this: # defaultdict(<class 'list'>, # {'/dev/sda': [PartitionInfo(device='/dev/sda', partition=None, mapper=None, uuid=None, mount=None), # PartitionInfo(device='/dev/sda', partition='/dev/sda1', mapper=None, uuid='B42B-7414', mount=None), # PartitionInfo(device='/dev/sda', partition='/dev/sda2', mapper=None, uuid='6DF8-D966', mount='/boot/efi'), # PartitionInfo(device='/dev/sda', partition='/dev/sda3', mapper=None, uuid='f6b575e4-0ec2-47ab-8d0a-9d677ac4fe3c', mount='/boot'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper=None, uuid='Lmx30A-U9qR-kDZF-WOju-zlOi-otrR-WNjh7j', mount=None), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-swap', uuid='00987200-7157-45d1-a233-90cbb22554aa', mount='[SWAP]'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-root', uuid='b53ea5c3-8771-4717-9d3d-ef9c5b18bbe4', mount='/'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-var', uuid='45aec3eb-68be-4eaa-bf79-de3f2a85c103', mount='/var'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-audit', uuid='339ee49c-0e45-4510-8447-55f46f2a3653', mount='/var/log/audit'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-tmp', uuid='b305d781-263f-4016-8422-301f61c11472', mount='/tmp'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-opt', uuid='5e7cbfb8-760e-4526-90d5-ab103ae626a5', mount='/opt'), # PartitionInfo(device='/dev/sda', partition='/dev/sda4', mapper='/dev/mapper/main-home', uuid='1b089fc0-f3a4-400b-955c-d3fa6b1e2a5f', mount='/home')], # '/dev/sdb': [PartitionInfo(device='/dev/sdb', partition=None, mapper=None, uuid=None, mount=None)]}) candidateDevs = [] formattedDevs = [] # determine candidate storage devices, which are any disks that do not have a mount point associated with # it in any way, (no partitions, mappings, etc. that are mounted) and is at least 100 gigabytes for device, entries in allDisks.items(): deviceMounts = list( set([par.mount for par in entries if par.mount is not None])) if (len(deviceMounts) == 0) and (GetDeviceSize(device) >= MINIMUM_CAPTURE_DEVICE_BYTES): candidateDevs.append(device) # sort candidate devices largest to smallest candidateDevs = sorted(candidateDevs, key=lambda x: GetDeviceSize(x), reverse=True) if debug: eprint( f"Device candidates: {[(x, sizeof_fmt(GetDeviceSize(x))) for x in candidateDevs]}" ) if len(candidateDevs) > 0: if args.encrypt: # create keyfile (will be on the encrypted system drive, and used to automatically unlock the encrypted capture drives) with open(CAPTURE_CRYPT_KEYFILE, 'wb') as f: f.write(os.urandom(4096)) os.chown(CAPTURE_CRYPT_KEYFILE, 0, 0) os.chmod(CAPTURE_CRYPT_KEYFILE, CAPTURE_CRYPT_KEYFILE_PERMS) # partition/format each candidate device for device in candidateDevs: # we only need at most two drives (one for pcap, one for zeek), or at least one if (len(formattedDevs) >= 2): break if (not args.interactive) or YesOrNo( f'Partition and format {device}{" (dry-run)" if args.dryrun else ""}?' ): if args.dryrun: eprint(f"Partitioning {device} (dry run only)...") eprint( f'\t/sbin/parted --script --align optimal {device} -- mklabel gpt \\\n\t\tunit mib mkpart primary 1 100%' ) ecode = 0 partedOut = [] else: # use parted to create a gpt partition table with a single partition consuming 100% of the disk minus one megabyte at the beginning if debug: eprint(f"Partitioning {device}...") ecode, partedOut = run_process( f'/sbin/parted --script --align optimal {device} -- mklabel gpt \\\n unit mib mkpart primary 1 100%', stdout=True, stderr=True, timeout=300) if debug: eprint(partedOut) if (ecode == 0): if debug: eprint(f"Success partitioning {device}") # get the list of partitions from the newly partitioned device (should be just one) _, fdiskOut = run_process(f'fdisk -l {device}') pars = [] parsList = False for line in fdiskOut: if debug: eprint(f"\t{line}") if (not parsList ) and fdisk_pars_begin_pattern.search(line): parsList = True elif parsList: match = fdisk_par_pattern.search(line) if match is not None: pars.append(match.group('device')) if len(pars) == 1: parDev = pars[0] parUuid = str(uuid.uuid4()) parMapperDev = None okToFormat = True if args.encrypt: okToFormat = False # remove this device from /etc/crypttab if os.path.isfile(CRYPTTAB_FILE): with fileinput.FileInput( CRYPTTAB_FILE, inplace=True, backup='.bak') as f: for line in f: line = line.rstrip("\n") if line.startswith( f"{CreateMapperName(parDev)}" ): if debug: eprint( f"removed {line} from {CRYPTTAB_FILE}" ) else: print(line) _, reloadOut = run_process( f"systemctl daemon-reload") # for good measure, run luksErase in case it was a previous luks volume if debug: eprint( f"Running crypsetup luksErase on {parDev}..." ) _, cryptOut = run_process( f"/sbin/cryptsetup --verbose --batch-mode luksErase {parDev}", stdout=True, stderr=True, timeout=600) if debug: for line in cryptOut: eprint(f"\t{line}") _, reloadOut = run_process( f"systemctl daemon-reload") # luks volume creation # format device as a luks volume if debug: eprint( f"Running crypsetup luksFormat on {device}..." ) ecode, cryptOut = run_process( f"/sbin/cryptsetup --verbose --batch-mode luksFormat {parDev} --uuid='{parUuid}' --key-file {CAPTURE_CRYPT_KEYFILE}", stdout=True, stderr=True, timeout=3600) if debug or (ecode != 0): for line in cryptOut: eprint(f"\t{line}") if (ecode == 0): # open the luks volume in /dev/mapper/ if debug: eprint( f"Running crypsetup luksOpen on {device}..." ) parMapperDev = CreateMapperDeviceName( parDev) ecode, cryptOut = run_process( f"/sbin/cryptsetup --verbose luksOpen {parDev} {CreateMapperName(parDev)} --key-file {CAPTURE_CRYPT_KEYFILE}", stdout=True, stderr=True, timeout=180) if debug or (ecode != 0): for line in cryptOut: eprint(f"\t{line}") if (ecode == 0): # we have everything we need for luks okToFormat = True else: eprint( f"Error {ecode} opening LUKS on {parDev}, giving up on {device}" ) else: eprint( f"Error {ecode} formatting LUKS on {parDev}, giving up on {device}" ) # format the partition as an XFS file system if okToFormat: if debug: eprint( f'Created {parDev}, assigning {parUuid}' ) if args.encrypt: formatCmd = f"/sbin/mkfs.xfs -f {parMapperDev}" else: formatCmd = f"/sbin/mkfs.xfs -f -m uuid='{parUuid}' {parDev}" if debug: eprint(f"Formatting: {formatCmd}") ecode, mkfsOut = run_process(formatCmd, stdout=True, stderr=True, timeout=3600) if debug: for line in mkfsOut: eprint(f"\t{line}") if (ecode == 0): eprint( f"Success formatting {parMapperDev if args.encrypt else parDev}" ) formattedDevs.append( PartitionInfo(device=device, partition=parDev, mapper=parMapperDev, uuid=parUuid, mount=None)) else: eprint( f"Error {ecode} formatting {formatPath}, giving up on {device}" ) else: eprint( f"Error partitioning {device}, unexpected partitions after running parted, giving up on {device}" ) elif (ecode != 0): eprint( f"Error {ecode} partitioning {device}, giving up on {device}" ) # now that we have formatted our device(s), decide where they're going to mount (these are already sorted) if len(formattedDevs) >= 2: formattedDevs[0].mount = os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_PCAP_DIR) formattedDevs[1].mount = os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_ZEEK_DIR) elif len(formattedDevs) == 1: formattedDevs[0].mount = CAPTURE_MOUNT_ROOT_PATH if debug: eprint(formattedDevs) # mountpoints are probably not already mounted, but this will make sure run_process( f"umount {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_PCAP_DIR)}" ) run_process( f"umount {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_ZEEK_DIR)}" ) run_process(f"umount {CAPTURE_MOUNT_ROOT_PATH}") _, reloadOut = run_process(f"systemctl daemon-reload") # clean out any previous fstab entries that might be interfering from previous configurations if Fstab.remove_by_mountpoint(os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_PCAP_DIR), path=FSTAB_FILE): if debug: eprint( f"Removed previous {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_PCAP_DIR)} mount from {FSTAB_FILE}" ) if Fstab.remove_by_mountpoint(os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_ZEEK_DIR), path=FSTAB_FILE): if debug: eprint( f"Removed previous {os.path.join(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_MOUNT_ZEEK_DIR)} mount from {FSTAB_FILE}" ) if Fstab.remove_by_mountpoint(CAPTURE_MOUNT_ROOT_PATH, path=FSTAB_FILE): if debug: eprint( f"Removed previous {CAPTURE_MOUNT_ROOT_PATH} mount from {FSTAB_FILE}" ) # reload tab files with systemctl _, reloadOut = run_process(f"systemctl daemon-reload") # get the GID of the group of the user(s) that will be doing the capture try: ecode, guidGetOut = run_process( f"getent group {CAPTURE_GROUP_OWNER}", stdout=True, stderr=True) if (ecode == 0) and (len(guidGetOut) > 0): netdevGuid = int(guidGetOut[0].split(':')[2]) else: netdevGuid = -1 except: netdevGuid = -1 # rmdir any mount directories that might be interfering from previous configurations if os.path.isdir(CAPTURE_MOUNT_ROOT_PATH): for root, dirs, files in os.walk(CAPTURE_MOUNT_ROOT_PATH, topdown=False): for name in dirs: if debug: eprint(f"Removing {os.path.join(root, name)}") os.rmdir(os.path.join(root, name)) if debug: eprint(f"Removing {CAPTURE_MOUNT_ROOT_PATH}") os.rmdir(CAPTURE_MOUNT_ROOT_PATH) if debug: eprint(f"Creating {CAPTURE_MOUNT_ROOT_PATH}") os.makedirs(CAPTURE_MOUNT_ROOT_PATH, exist_ok=True) os.chown(CAPTURE_MOUNT_ROOT_PATH, -1, netdevGuid) os.chmod(CAPTURE_MOUNT_ROOT_PATH, CAPTURE_DIR_PERMS) # add crypttab entries if args.encrypt: with open(CRYPTTAB_FILE, 'a' if os.path.isfile(CRYPTTAB_FILE) else 'w') as f: for par in formattedDevs: crypttabLine = f"{CreateMapperName(par.partition)} UUID={par.uuid} {CAPTURE_CRYPT_KEYFILE} luks\n" f.write(crypttabLine) if debug: eprint(f'Added "{crypttabLine}" to {CRYPTTAB_FILE}') # recreate mount directories and add fstab entries for par in formattedDevs: if debug: eprint(f"Creating {par.mount}") os.makedirs(par.mount, exist_ok=True) if args.encrypt: entry = Fstab.add( device=f"{par.mapper}", mountpoint=par.mount, options= f"defaults,inode64,noatime,rw,auto,user,x-systemd.device-timeout=600s", fs_passno=2, filesystem='xfs', path=FSTAB_FILE) else: entry = Fstab.add( device=f"UUID={par.uuid}", mountpoint=par.mount, options= f"defaults,inode64,noatime,rw,auto,user,x-systemd.device-timeout=600s", fs_passno=2, filesystem='xfs', path=FSTAB_FILE) eprint(f'Added "{entry}" to {FSTAB_FILE} for {par.partition}') # reload tab files with systemctl _, reloadOut = run_process(f"systemctl daemon-reload") # mount the partitions and create a directory with user permissions for par in formattedDevs: ecode, mountOut = run_process(f"mount {par.mount}") if (ecode == 0): if debug: eprint(f'Mounted {par.partition} at {par.mount}') userDirs = [] if par.mount == CAPTURE_MOUNT_ROOT_PATH: # only one drive, so we're mounted at /capture, create user directories for CAPTURE_MOUNT_ZEEK_DIR and CAPTURE_MOUNT_PCAP_DIR userDirs.append( os.path.join(par.mount, CAPTURE_MOUNT_PCAP_DIR)) userDirs.append( os.path.join(par.mount, CAPTURE_MOUNT_ZEEK_DIR)) else: # we're mounted somewhere *underneath* /capture, so create a user-writeable subdirectory where we are userDirs.append(os.path.join(par.mount, 'capture')) # set permissions on user dirs pcapDir = None zeekDir = None for userDir in userDirs: os.makedirs(userDir, exist_ok=True) os.chown(userDir, CAPTURE_USER_UID, netdevGuid) os.chmod(userDir, CAPTURE_SUBDIR_PERMS) if debug: eprint( f'Created "{userDir}" for writing by capture user') if f"{os.path.sep}{CAPTURE_MOUNT_PCAP_DIR}{os.path.sep}" in userDir: pcapDir = userDir elif f"{os.path.sep}{CAPTURE_MOUNT_ZEEK_DIR}{os.path.sep}" in userDir: zeekDir = userDir # replace capture paths in-place in SENSOR_CAPTURE_CONFIG if os.path.isfile(SENSOR_CAPTURE_CONFIG): capture_re = re.compile( r"\b(?P<key>PCAP_PATH|ZEEK_LOG_PATH)\s*=\s*.*?$") with fileinput.FileInput(SENSOR_CAPTURE_CONFIG, inplace=True, backup='.bak') as f: for line in f: line = line.rstrip("\n") log_path_match = capture_re.search(line) if (log_path_match is not None): if (log_path_match.group('key') == 'PCAP_PATH') and (pcapDir is not None): print( capture_re.sub(r"\1=%s" % pcapDir, line)) elif (log_path_match.group('key') == 'ZEEK_LOG_PATH') and (zeekDir is not None): print( capture_re.sub(r"\1=%s" % zeekDir, line)) else: print(line) else: print(line) else: eprint(f"Error {ecode} mounting {par.partition}") else: eprint( f"Could not find any unmounted devices greater than 100GB, giving up" )