class Inspector(object): MODE = ['configuration', 'payload', 'all'] IGNORE_MOUNTS = [ "proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon" ] IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"] IGNORE_PATHS = [ "/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home" ] def __init__(self, db_path=None, pid_file=None): # Configured path if not db_path and '__salt__' in globals(): db_path = globals().get('__salt__')['config.get']('inspector.db', '') if not db_path: raise InspectorSnapshotException( "Inspector database location is not configured yet in minion.") self.dbfile = db_path self.db = DBHandle(self.dbfile) self.db.open() if not pid_file and '__salt__' in globals(): pid_file = globals().get('__salt__')['config.get']('inspector.pid', '') if not pid_file: raise InspectorSnapshotException( "Inspector PID file location is not configured yet in minion.") self.pidfile = pid_file def _syscall(self, command, input=None, env=None, *params): ''' Call an external system command. ''' return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT, env=env or os.environ).communicate(input=input) def _get_cfg_pkgs(self): ''' Get packages with configuration files. ''' out, err = self._syscall('rpm', None, None, '-qa', '--configfiles', '--queryformat', '%{name}-%{version}-%{release}\\n') data = dict() pkg_name = None pkg_configs = [] out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line: continue if not line.startswith("/"): if pkg_name and pkg_configs: data[pkg_name] = pkg_configs pkg_name = line pkg_configs = [] else: pkg_configs.append(line) if pkg_name and pkg_configs: data[pkg_name] = pkg_configs return data def _get_changed_cfg_pkgs(self, data): ''' Filter out unchanged packages. ''' f_data = dict() for pkg_name, pkg_files in data.items(): cfgs = list() out, err = self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', '--nosignature', '--nomtime', '--nolinkto', pkg_name) out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line or line.find(" c ") < 0 or line.split(" ")[0].find( "5") < 0: continue cfg_file = line.split(" ")[-1] if cfg_file in pkg_files: cfgs.append(cfg_file) if cfgs: f_data[pkg_name] = cfgs return f_data def _save_cfg_pkgs(self, data): ''' Save configuration packages. ''' for table in ["inspector_pkg", "inspector_pkg_cfg_files"]: self.db.flush(table) pkg_id = 0 pkg_cfg_id = 0 for pkg_name, pkg_configs in data.items(): self.db.cursor.execute( "INSERT INTO inspector_pkg (id, name) VALUES (?, ?)", (pkg_id, pkg_name)) for pkg_config in pkg_configs: self.db.cursor.execute( "INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)", (pkg_cfg_id, pkg_id, pkg_config)) pkg_cfg_id += 1 pkg_id += 1 self.db.connection.commit() def _save_payload(self, files, directories, links): ''' Save payload (unmanaged files) ''' idx = 0 for p_type, p_list in ( ('f', files), ('d', directories), ( 'l', links, ), ): for p_obj in p_list: stats = os.stat(p_obj) self.db.cursor.execute( "INSERT INTO inspector_payload " "(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)" "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size, stats.st_atime, stats.st_mtime, stats.st_ctime)) idx += 1 self.db.connection.commit() def _get_managed_files(self): ''' Build a in-memory data of all managed files. ''' dirs = set() links = set() files = set() cmd = __salt__['cmd.run_stdout']('rpm -qlav') for line in cmd: line = line.strip() if not line: continue line = line.replace("\t", " ").split(" ") if line[0][0] == "d": dirs.add(line[-1]) elif line[0][0] == "l": links.add(line[-1]) elif line[0][0] == "-": files.add(line[-1]) return sorted(files), sorted(dirs), sorted(links) def _get_all_files(self, path, *exclude): ''' Walk implementation. Version in python 2.x and 3.x works differently. ''' files = list() dirs = list() links = list() for obj in os.listdir(path): obj = os.path.join(path, obj) valid = True for ex_obj in exclude: if obj.startswith(str(ex_obj)): valid = False continue if not valid or not os.path.exists(obj): continue mode = os.lstat(obj).st_mode if stat.S_ISLNK(mode): links.append(obj) elif stat.S_ISDIR(mode): dirs.append(obj) f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude) files.extend(f_obj) dirs.extend(d_obj) links.extend(l_obj) elif stat.S_ISREG(mode): files.append(obj) return sorted(files), sorted(dirs), sorted(links) def _get_unmanaged_files(self, managed, system_all): ''' Get the intersection between all files and managed files. ''' def intr(src, data): out = set() for d_el in data: if d_el not in src: out.add(d_el) return out m_files, m_dirs, m_links = managed s_files, s_dirs, s_links = system_all return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted( intr(m_links, s_links)) def _scan_payload(self): ''' Scan the system. ''' # Get ignored points allowed = list() self.db.cursor.execute("SELECT path FROM inspector_allowed") for alwd_path in self.db.cursor.fetchall(): if os.path.exists(alwd_path[0]): allowed.append(alwd_path[0]) ignored = list() if not allowed: self.db.cursor.execute("SELECT path FROM inspector_ignored") for ign_path in self.db.cursor.fetchall(): ignored.append(ign_path[0]) all_files = list() all_dirs = list() all_links = list() for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]: if entry_path[0] != "/": entry_path = "/{0}".format(entry_path) if entry_path in ignored: continue e_files, e_dirs, e_links = self._get_all_files( entry_path, *ignored) all_files.extend(e_files) all_dirs.extend(e_dirs) all_links.extend(e_links) return self._get_unmanaged_files(self._get_managed_files(), ( all_files, all_dirs, all_links, )) def _prepare_full_scan(self, **kwargs): ''' Prepare full system scan by setting up the database etc. ''' # TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried. self.db.purge() # Add ignored filesystems ignored_fs = set() ignored_fs |= set(self.IGNORE_PATHS) mounts = fsutils._get_mounts() for device, data in mounts.items(): if device in self.IGNORE_MOUNTS: for mpt in data: ignored_fs.add(mpt['mount_point']) continue for mpt in data: if mpt['type'] in self.IGNORE_FS_TYPES: ignored_fs.add(mpt['mount_point']) # Remove leafs of ignored filesystems ignored_all = list() for entry in sorted(list(ignored_fs)): valid = True for e_entry in ignored_all: if entry.startswith(e_entry): valid = False break if valid: ignored_all.append(entry) # Save to the database for further scan for ignored_dir in ignored_all: self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir, )) # Add allowed filesystems (overrides all above at full scan) allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm] for allowed_dir in allowed: self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir, )) self.db.connection.commit() return ignored_all def snapshot(self, mode): ''' Take a snapshot of the system. ''' # TODO: Mode self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs())) self._save_payload(*self._scan_payload()) def request_snapshot(self, mode, priority=19, **kwargs): ''' Take a snapshot of the system. ''' if mode not in self.MODE: raise InspectorSnapshotException( "Unknown mode: '{0}'".format(mode)) self._prepare_full_scan(**kwargs) os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format( priority, __file__, self.pidfile, self.dbfile, mode))
class Inspector(object): DEFAULT_MINION_CONFIG_PATH = '/etc/salt/minion' MODE = ['configuration', 'payload', 'all'] IGNORE_MOUNTS = ["proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"] IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"] IGNORE_PATHS = ["/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home"] def __init__(self, db_path=None, pid_file=None): # Configured path if not db_path and '__salt__' in globals(): db_path = globals().get('__salt__')['config.get']('inspector.db', '') if not db_path: raise InspectorSnapshotException('Inspector database location is not configured yet in minion.\n' 'Add "inspector.db: /path/to/cache" in "/etc/salt/minion".') self.dbfile = db_path self.db = DBHandle(self.dbfile) self.db.open() if not pid_file and '__salt__' in globals(): pid_file = globals().get('__salt__')['config.get']('inspector.pid', '') if not pid_file: raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.\n" 'Add "inspector.pid: /path/to/pids in "/etc/salt/minion".') self.pidfile = pid_file def _syscall(self, command, input=None, env=None, *params): ''' Call an external system command. ''' return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT, env=env or os.environ).communicate(input=input) def _get_cfg_pkgs(self): ''' Package scanner switcher between the platforms. :return: ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_cfg_pkgs_dpkg() elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_cfg_pkgs_rpm() else: return dict() def __get_cfg_pkgs_dpkg(self): ''' Get packages with configuration files on Dpkg systems. :return: ''' # Get list of all available packages data = dict() for pkg_name in salt.utils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${binary:Package}\\n")[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue data[pkg_name] = list() for pkg_cfg_item in salt.utils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${Conffiles}\\n", pkg_name)[0]).split(os.linesep): pkg_cfg_item = pkg_cfg_item.strip() if not pkg_cfg_item: continue pkg_cfg_file, pkg_cfg_sum = pkg_cfg_item.strip().split(" ", 1) data[pkg_name].append(pkg_cfg_file) # Dpkg meta data is unreliable. Check every package # and remove which actually does not have config files. if not data[pkg_name]: data.pop(pkg_name) return data def __get_cfg_pkgs_rpm(self): ''' Get packages with configuration files on RPM systems. ''' out, err = self._syscall('rpm', None, None, '-qa', '--configfiles', '--queryformat', '%{name}-%{version}-%{release}\\n') data = dict() pkg_name = None pkg_configs = [] out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line: continue if not line.startswith("/"): if pkg_name and pkg_configs: data[pkg_name] = pkg_configs pkg_name = line pkg_configs = [] else: pkg_configs.append(line) if pkg_name and pkg_configs: data[pkg_name] = pkg_configs return data def _get_changed_cfg_pkgs(self, data): ''' Filter out unchanged packages on the Debian or RPM systems. :param data: Structure {package-name -> [ file .. file1 ]} :return: Same structure as data, except only files that were changed. ''' f_data = dict() for pkg_name, pkg_files in data.items(): cfgs = list() cfg_data = list() if self.grains_core.os_data().get('os_family') == 'Debian': cfg_data = salt.utils.to_str(self._syscall("dpkg", None, None, '--verify', pkg_name)[0]).split(os.linesep) elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: cfg_data = salt.utils.to_str(self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', '--nosignature', '--nomtime', '--nolinkto', pkg_name)[0]).split(os.linesep) for line in cfg_data: line = line.strip() if not line or line.find(" c ") < 0 or line.split(" ")[0].find("5") < 0: continue cfg_file = line.split(" ")[-1] if cfg_file in pkg_files: cfgs.append(cfg_file) if cfgs: f_data[pkg_name] = cfgs return f_data def _save_cfg_pkgs(self, data): ''' Save configuration packages. ''' for table in ["inspector_pkg", "inspector_pkg_cfg_files"]: self.db.flush(table) pkg_id = 0 pkg_cfg_id = 0 for pkg_name, pkg_configs in data.items(): self.db.cursor.execute("INSERT INTO inspector_pkg (id, name) VALUES (?, ?)", (pkg_id, pkg_name)) for pkg_config in pkg_configs: self.db.cursor.execute("INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)", (pkg_cfg_id, pkg_id, pkg_config)) pkg_cfg_id += 1 pkg_id += 1 self.db.connection.commit() def _save_payload(self, files, directories, links): ''' Save payload (unmanaged files) ''' idx = 0 for p_type, p_list in (('f', files), ('d', directories), ('l', links,),): for p_obj in p_list: stats = os.stat(p_obj) self.db.cursor.execute("INSERT INTO inspector_payload " "(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)" "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size, stats.st_atime, stats.st_mtime, stats.st_ctime)) idx += 1 self.db.connection.commit() def _get_managed_files(self): ''' Build a in-memory data of all managed files. ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_managed_files_dpkg() elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_managed_files_rpm() return list(), list(), list() def __get_managed_files_dpkg(self): ''' Get a list of all system files, belonging to the Debian package manager. ''' dirs = set() links = set() files = set() for pkg_name in salt.utils.to_str(self._syscall("dpkg-query", None, None, '-Wf', '${binary:Package}\\n')[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue for resource in salt.utils.to_str(self._syscall("dpkg", None, None, '-L', pkg_name)[0]).split(os.linesep): resource = resource.strip() if not resource or resource in ['/', './', '.']: continue if os.path.isdir(resource): dirs.add(resource) elif os.path.islink(resource): links.add(resource) elif os.path.isfile(resource): files.add(resource) return sorted(files), sorted(dirs), sorted(links) def __get_managed_files_rpm(self): ''' Get a list of all system files, belonging to the RedHat package manager. ''' dirs = set() links = set() files = set() for line in salt.utils.to_str(self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep): line = line.strip() if not line: continue line = line.replace("\t", " ").split(" ") if line[0][0] == "d": dirs.add(line[-1]) elif line[0][0] == "l": links.add(line[-1]) elif line[0][0] == "-": files.add(line[-1]) return sorted(files), sorted(dirs), sorted(links) def _get_all_files(self, path, *exclude): ''' Walk implementation. Version in python 2.x and 3.x works differently. ''' files = list() dirs = list() links = list() for obj in os.listdir(path): obj = os.path.join(path, obj) valid = True for ex_obj in exclude: if obj.startswith(str(ex_obj)): valid = False continue if not valid or not os.path.exists(obj): continue if os.path.islink(obj): links.append(obj) elif os.path.isdir(obj): dirs.append(obj) f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude) files.extend(f_obj) dirs.extend(d_obj) links.extend(l_obj) elif os.path.isfile(obj): files.append(obj) return sorted(files), sorted(dirs), sorted(links) def _get_unmanaged_files(self, managed, system_all): ''' Get the intersection between all files and managed files. ''' def intr(src, data): out = set() for d_el in data: if d_el not in src: out.add(d_el) return out m_files, m_dirs, m_links = managed s_files, s_dirs, s_links = system_all return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted(intr(m_links, s_links)) def _scan_payload(self): ''' Scan the system. ''' # Get ignored points allowed = list() self.db.cursor.execute("SELECT path FROM inspector_allowed") for alwd_path in self.db.cursor.fetchall(): if os.path.exists(alwd_path[0]): allowed.append(alwd_path[0]) ignored = list() if not allowed: self.db.cursor.execute("SELECT path FROM inspector_ignored") for ign_path in self.db.cursor.fetchall(): ignored.append(ign_path[0]) all_files = list() all_dirs = list() all_links = list() for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]: if entry_path[0] != "/": entry_path = "/{0}".format(entry_path) if entry_path in ignored or os.path.islink(entry_path): continue e_files, e_dirs, e_links = self._get_all_files(entry_path, *ignored) all_files.extend(e_files) all_dirs.extend(e_dirs) all_links.extend(e_links) return self._get_unmanaged_files(self._get_managed_files(), (all_files, all_dirs, all_links,)) def _prepare_full_scan(self, **kwargs): ''' Prepare full system scan by setting up the database etc. ''' # TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried. self.db.purge() # Add ignored filesystems ignored_fs = set() ignored_fs |= set(self.IGNORE_PATHS) mounts = fsutils._get_mounts() for device, data in mounts.items(): if device in self.IGNORE_MOUNTS: for mpt in data: ignored_fs.add(mpt['mount_point']) continue for mpt in data: if mpt['type'] in self.IGNORE_FS_TYPES: ignored_fs.add(mpt['mount_point']) # Remove leafs of ignored filesystems ignored_all = list() for entry in sorted(list(ignored_fs)): valid = True for e_entry in ignored_all: if entry.startswith(e_entry): valid = False break if valid: ignored_all.append(entry) # Save to the database for further scan for ignored_dir in ignored_all: self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir,)) # Add allowed filesystems (overrides all above at full scan) allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm] for allowed_dir in allowed: self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir,)) self.db.connection.commit() return ignored_all def _init_env(self): ''' Initialize some Salt environment. ''' from salt.config import minion_config from salt.grains import core as g_core g_core.__opts__ = minion_config(self.DEFAULT_MINION_CONFIG_PATH) self.grains_core = g_core def snapshot(self, mode): ''' Take a snapshot of the system. ''' self._init_env() self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs())) self._save_payload(*self._scan_payload()) def request_snapshot(self, mode, priority=19, **kwargs): ''' Take a snapshot of the system. ''' if mode not in self.MODE: raise InspectorSnapshotException("Unknown mode: '{0}'".format(mode)) self._prepare_full_scan(**kwargs) os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format( priority, __file__, self.pidfile, self.dbfile, mode))
class Inspector(object): DEFAULT_MINION_CONFIG_PATH = '/etc/salt/minion' MODE = ['configuration', 'payload', 'all'] IGNORE_MOUNTS = [ "proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon" ] IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"] IGNORE_PATHS = [ "/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home" ] def __init__(self, db_path=None, pid_file=None): # Configured path if not db_path and '__salt__' in globals(): db_path = globals().get('__salt__')['config.get']('inspector.db', '') if not db_path: raise InspectorSnapshotException( 'Inspector database location is not configured yet in minion.\n' 'Add "inspector.db: /path/to/cache" in "/etc/salt/minion".') self.dbfile = db_path self.db = DBHandle(self.dbfile) self.db.open() if not pid_file and '__salt__' in globals(): pid_file = globals().get('__salt__')['config.get']('inspector.pid', '') if not pid_file: raise InspectorSnapshotException( "Inspector PID file location is not configured yet in minion.\n" 'Add "inspector.pid: /path/to/pids in "/etc/salt/minion".') self.pidfile = pid_file def _syscall(self, command, input=None, env=None, *params): ''' Call an external system command. ''' return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT, env=env or os.environ).communicate(input=input) def _get_cfg_pkgs(self): ''' Package scanner switcher between the platforms. :return: ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_cfg_pkgs_dpkg() elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_cfg_pkgs_rpm() else: return dict() def __get_cfg_pkgs_dpkg(self): ''' Get packages with configuration files on Dpkg systems. :return: ''' # Get list of all available packages data = dict() for pkg_name in salt.utils.to_str( self._syscall('dpkg-query', None, None, '-Wf', "${binary:Package}\\n")[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue data[pkg_name] = list() for pkg_cfg_item in salt.utils.to_str( self._syscall('dpkg-query', None, None, '-Wf', "${Conffiles}\\n", pkg_name)[0]).split(os.linesep): pkg_cfg_item = pkg_cfg_item.strip() if not pkg_cfg_item: continue pkg_cfg_file, pkg_cfg_sum = pkg_cfg_item.strip().split(" ", 1) data[pkg_name].append(pkg_cfg_file) # Dpkg meta data is unreliable. Check every package # and remove which actually does not have config files. if not data[pkg_name]: data.pop(pkg_name) return data def __get_cfg_pkgs_rpm(self): ''' Get packages with configuration files on RPM systems. ''' out, err = self._syscall('rpm', None, None, '-qa', '--configfiles', '--queryformat', '%{name}-%{version}-%{release}\\n') data = dict() pkg_name = None pkg_configs = [] out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line: continue if not line.startswith("/"): if pkg_name and pkg_configs: data[pkg_name] = pkg_configs pkg_name = line pkg_configs = [] else: pkg_configs.append(line) if pkg_name and pkg_configs: data[pkg_name] = pkg_configs return data def _get_changed_cfg_pkgs(self, data): ''' Filter out unchanged packages on the Debian or RPM systems. :param data: Structure {package-name -> [ file .. file1 ]} :return: Same structure as data, except only files that were changed. ''' f_data = dict() for pkg_name, pkg_files in data.items(): cfgs = list() cfg_data = list() if self.grains_core.os_data().get('os_family') == 'Debian': cfg_data = salt.utils.to_str( self._syscall("dpkg", None, None, '--verify', pkg_name)[0]).split(os.linesep) elif self.grains_core.os_data().get('os_family') in [ 'Suse', 'redhat' ]: cfg_data = salt.utils.to_str( self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', '--nosignature', '--nomtime', '--nolinkto', pkg_name)[0]).split(os.linesep) for line in cfg_data: line = line.strip() if not line or line.find(" c ") < 0 or line.split(" ")[0].find( "5") < 0: continue cfg_file = line.split(" ")[-1] if cfg_file in pkg_files: cfgs.append(cfg_file) if cfgs: f_data[pkg_name] = cfgs return f_data def _save_cfg_pkgs(self, data): ''' Save configuration packages. ''' for table in ["inspector_pkg", "inspector_pkg_cfg_files"]: self.db.flush(table) pkg_id = 0 pkg_cfg_id = 0 for pkg_name, pkg_configs in data.items(): self.db.cursor.execute( "INSERT INTO inspector_pkg (id, name) VALUES (?, ?)", (pkg_id, pkg_name)) for pkg_config in pkg_configs: self.db.cursor.execute( "INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)", (pkg_cfg_id, pkg_id, pkg_config)) pkg_cfg_id += 1 pkg_id += 1 self.db.connection.commit() def _save_payload(self, files, directories, links): ''' Save payload (unmanaged files) ''' idx = 0 for p_type, p_list in ( ('f', files), ('d', directories), ( 'l', links, ), ): for p_obj in p_list: stats = os.stat(p_obj) self.db.cursor.execute( "INSERT INTO inspector_payload " "(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)" "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size, stats.st_atime, stats.st_mtime, stats.st_ctime)) idx += 1 self.db.connection.commit() def _get_managed_files(self): ''' Build a in-memory data of all managed files. ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_managed_files_dpkg() elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_managed_files_rpm() return list(), list(), list() def __get_managed_files_dpkg(self): ''' Get a list of all system files, belonging to the Debian package manager. ''' dirs = set() links = set() files = set() for pkg_name in salt.utils.to_str( self._syscall("dpkg-query", None, None, '-Wf', '${binary:Package}\\n')[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue for resource in salt.utils.to_str( self._syscall("dpkg", None, None, '-L', pkg_name)[0]).split(os.linesep): resource = resource.strip() if not resource or resource in ['/', './', '.']: continue if os.path.isdir(resource): dirs.add(resource) elif os.path.islink(resource): links.add(resource) elif os.path.isfile(resource): files.add(resource) return sorted(files), sorted(dirs), sorted(links) def __get_managed_files_rpm(self): ''' Get a list of all system files, belonging to the RedHat package manager. ''' dirs = set() links = set() files = set() for line in salt.utils.to_str( self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep): line = line.strip() if not line: continue line = line.replace("\t", " ").split(" ") if line[0][0] == "d": dirs.add(line[-1]) elif line[0][0] == "l": links.add(line[-1]) elif line[0][0] == "-": files.add(line[-1]) return sorted(files), sorted(dirs), sorted(links) def _get_all_files(self, path, *exclude): ''' Walk implementation. Version in python 2.x and 3.x works differently. ''' files = list() dirs = list() links = list() for obj in os.listdir(path): obj = os.path.join(path, obj) valid = True for ex_obj in exclude: if obj.startswith(str(ex_obj)): valid = False continue if not valid or not os.path.exists(obj): continue if os.path.islink(obj): links.append(obj) elif os.path.isdir(obj): dirs.append(obj) f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude) files.extend(f_obj) dirs.extend(d_obj) links.extend(l_obj) elif os.path.isfile(obj): files.append(obj) return sorted(files), sorted(dirs), sorted(links) def _get_unmanaged_files(self, managed, system_all): ''' Get the intersection between all files and managed files. ''' def intr(src, data): out = set() for d_el in data: if d_el not in src: out.add(d_el) return out m_files, m_dirs, m_links = managed s_files, s_dirs, s_links = system_all return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted( intr(m_links, s_links)) def _scan_payload(self): ''' Scan the system. ''' # Get ignored points allowed = list() self.db.cursor.execute("SELECT path FROM inspector_allowed") for alwd_path in self.db.cursor.fetchall(): if os.path.exists(alwd_path[0]): allowed.append(alwd_path[0]) ignored = list() if not allowed: self.db.cursor.execute("SELECT path FROM inspector_ignored") for ign_path in self.db.cursor.fetchall(): ignored.append(ign_path[0]) all_files = list() all_dirs = list() all_links = list() for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]: if entry_path[0] != "/": entry_path = "/{0}".format(entry_path) if entry_path in ignored or os.path.islink(entry_path): continue e_files, e_dirs, e_links = self._get_all_files( entry_path, *ignored) all_files.extend(e_files) all_dirs.extend(e_dirs) all_links.extend(e_links) return self._get_unmanaged_files(self._get_managed_files(), ( all_files, all_dirs, all_links, )) def _prepare_full_scan(self, **kwargs): ''' Prepare full system scan by setting up the database etc. ''' # TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried. self.db.purge() # Add ignored filesystems ignored_fs = set() ignored_fs |= set(self.IGNORE_PATHS) mounts = fsutils._get_mounts() for device, data in mounts.items(): if device in self.IGNORE_MOUNTS: for mpt in data: ignored_fs.add(mpt['mount_point']) continue for mpt in data: if mpt['type'] in self.IGNORE_FS_TYPES: ignored_fs.add(mpt['mount_point']) # Remove leafs of ignored filesystems ignored_all = list() for entry in sorted(list(ignored_fs)): valid = True for e_entry in ignored_all: if entry.startswith(e_entry): valid = False break if valid: ignored_all.append(entry) # Save to the database for further scan for ignored_dir in ignored_all: self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir, )) # Add allowed filesystems (overrides all above at full scan) allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm] for allowed_dir in allowed: self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir, )) self.db.connection.commit() return ignored_all def _init_env(self): ''' Initialize some Salt environment. ''' from salt.config import minion_config from salt.grains import core as g_core g_core.__opts__ = minion_config(self.DEFAULT_MINION_CONFIG_PATH) self.grains_core = g_core def snapshot(self, mode): ''' Take a snapshot of the system. ''' self._init_env() self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs())) self._save_payload(*self._scan_payload()) def request_snapshot(self, mode, priority=19, **kwargs): ''' Take a snapshot of the system. ''' if mode not in self.MODE: raise InspectorSnapshotException( "Unknown mode: '{0}'".format(mode)) self._prepare_full_scan(**kwargs) os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format( priority, __file__, self.pidfile, self.dbfile, mode))
class Inspector(object): MODE = ['configuration', 'payload', 'all'] IGNORE_MOUNTS = ["proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"] IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"] IGNORE_PATHS = ["/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home"] def __init__(self, db_path=None, pid_file=None): # Configured path if not db_path and '__salt__' in globals(): db_path = globals().get('__salt__')['config.get']('inspector.db', '') if not db_path: raise InspectorSnapshotException("Inspector database location is not configured yet in minion.") self.dbfile = db_path self.db = DBHandle(self.dbfile) self.db.open() if not pid_file and '__salt__' in globals(): pid_file = globals().get('__salt__')['config.get']('inspector.pid', '') if not pid_file: raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.") self.pidfile = pid_file def _syscall(self, command, input=None, env=None, *params): ''' Call an external system command. ''' return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT, env=env or os.environ).communicate(input=input) def _get_cfg_pkgs(self): ''' Get packages with configuration files. ''' out, err = self._syscall('rpm', None, None, '-qa', '--configfiles', '--queryformat', '%{name}-%{version}-%{release}\\n') data = dict() pkg_name = None pkg_configs = [] out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line: continue if not line.startswith("/"): if pkg_name and pkg_configs: data[pkg_name] = pkg_configs pkg_name = line pkg_configs = [] else: pkg_configs.append(line) if pkg_name and pkg_configs: data[pkg_name] = pkg_configs return data def _get_changed_cfg_pkgs(self, data): ''' Filter out unchanged packages. ''' f_data = dict() for pkg_name, pkg_files in data.items(): cfgs = list() out, err = self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', '--nosignature', '--nomtime', '--nolinkto', pkg_name) out = salt.utils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line or line.find(" c ") < 0 or line.split(" ")[0].find("5") < 0: continue cfg_file = line.split(" ")[-1] if cfg_file in pkg_files: cfgs.append(cfg_file) if cfgs: f_data[pkg_name] = cfgs return f_data def _save_cfg_pkgs(self, data): ''' Save configuration packages. ''' for table in ["inspector_pkg", "inspector_pkg_cfg_files"]: self.db.flush(table) pkg_id = 0 pkg_cfg_id = 0 for pkg_name, pkg_configs in data.items(): self.db.cursor.execute("INSERT INTO inspector_pkg (id, name) VALUES (?, ?)", (pkg_id, pkg_name)) for pkg_config in pkg_configs: self.db.cursor.execute("INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)", (pkg_cfg_id, pkg_id, pkg_config)) pkg_cfg_id += 1 pkg_id += 1 self.db.connection.commit() def _save_payload(self, files, directories, links): ''' Save payload (unmanaged files) ''' idx = 0 for p_type, p_list in (('f', files), ('d', directories), ('l', links,),): for p_obj in p_list: stats = os.stat(p_obj) self.db.cursor.execute("INSERT INTO inspector_payload " "(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)" "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size, stats.st_atime, stats.st_mtime, stats.st_ctime)) idx += 1 self.db.connection.commit() def _get_managed_files(self): ''' Build a in-memory data of all managed files. ''' dirs = set() links = set() files = set() cmd = __salt__['cmd.run_stdout']('rpm -qlav') for line in cmd: line = line.strip() if not line: continue line = line.replace("\t", " ").split(" ") if line[0][0] == "d": dirs.add(line[-1]) elif line[0][0] == "l": links.add(line[-1]) elif line[0][0] == "-": files.add(line[-1]) return sorted(files), sorted(dirs), sorted(links) def _get_all_files(self, path, *exclude): ''' Walk implementation. Version in python 2.x and 3.x works differently. ''' files = list() dirs = list() links = list() for obj in os.listdir(path): obj = os.path.join(path, obj) valid = True for ex_obj in exclude: if obj.startswith(str(ex_obj)): valid = False continue if not valid or not os.path.exists(obj): continue mode = os.lstat(obj).st_mode if stat.S_ISLNK(mode): links.append(obj) elif stat.S_ISDIR(mode): dirs.append(obj) f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude) files.extend(f_obj) dirs.extend(d_obj) links.extend(l_obj) elif stat.S_ISREG(mode): files.append(obj) return sorted(files), sorted(dirs), sorted(links) def _get_unmanaged_files(self, managed, system_all): ''' Get the intersection between all files and managed files. ''' def intr(src, data): out = set() for d_el in data: if d_el not in src: out.add(d_el) return out m_files, m_dirs, m_links = managed s_files, s_dirs, s_links = system_all return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted(intr(m_links, s_links)) def _scan_payload(self): ''' Scan the system. ''' # Get ignored points allowed = list() self.db.cursor.execute("SELECT path FROM inspector_allowed") for alwd_path in self.db.cursor.fetchall(): if os.path.exists(alwd_path[0]): allowed.append(alwd_path[0]) ignored = list() if not allowed: self.db.cursor.execute("SELECT path FROM inspector_ignored") for ign_path in self.db.cursor.fetchall(): ignored.append(ign_path[0]) all_files = list() all_dirs = list() all_links = list() for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]: if entry_path[0] != "/": entry_path = "/{0}".format(entry_path) if entry_path in ignored: continue e_files, e_dirs, e_links = self._get_all_files(entry_path, *ignored) all_files.extend(e_files) all_dirs.extend(e_dirs) all_links.extend(e_links) return self._get_unmanaged_files(self._get_managed_files(), (all_files, all_dirs, all_links,)) def _prepare_full_scan(self, **kwargs): ''' Prepare full system scan by setting up the database etc. ''' # TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried. self.db.purge() # Add ignored filesystems ignored_fs = set() ignored_fs |= set(self.IGNORE_PATHS) mounts = fsutils._get_mounts() for device, data in mounts.items(): if device in self.IGNORE_MOUNTS: for mpt in data: ignored_fs.add(mpt['mount_point']) continue for mpt in data: if mpt['type'] in self.IGNORE_FS_TYPES: ignored_fs.add(mpt['mount_point']) # Remove leafs of ignored filesystems ignored_all = list() for entry in sorted(list(ignored_fs)): valid = True for e_entry in ignored_all: if entry.startswith(e_entry): valid = False break if valid: ignored_all.append(entry) # Save to the database for further scan for ignored_dir in ignored_all: self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir,)) # Add allowed filesystems (overrides all above at full scan) allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm] for allowed_dir in allowed: self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir,)) self.db.connection.commit() return ignored_all def snapshot(self, mode): ''' Take a snapshot of the system. ''' # TODO: Mode self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs())) self._save_payload(*self._scan_payload()) def request_snapshot(self, mode, priority=19, **kwargs): ''' Take a snapshot of the system. ''' if mode not in self.MODE: raise InspectorSnapshotException("Unknown mode: '{0}'".format(mode)) self._prepare_full_scan(**kwargs) os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format( priority, __file__, self.pidfile, self.dbfile, mode))