def __init__(self, scope):
'''
Constructor.
:param scope:
:return:
'''
if scope not in self.SCOPES:
raise InspectorQueryException(
"Unknown scope: {0}. Must be one of: {1}".format(
repr(scope), ", ".join(self.SCOPES)))
self.scope = '_' + scope
self.db = DBHandle(globals()['__salt__']['config.get']('inspector.db',
''))
self.local_identity = dict()
def __init__(self, db_path=None, pid_file=None):
# Configured path
if not db_path and '__salt__' in globals():
db_path = globals().get('__salt__')['config.get']('inspector.db', '')
if not db_path:
raise InspectorSnapshotException("Inspector database location is not configured yet in minion.")
self.dbfile = db_path
self.db = DBHandle(self.dbfile)
self.db.open()
if not pid_file and '__salt__' in globals():
pid_file = globals().get('__salt__')['config.get']('inspector.pid', '')
if not pid_file:
raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.")
self.pidfile = pid_file
def __init__(self, scope):
'''
Constructor.
:param scope:
:return:
'''
if scope not in self.SCOPES:
raise InspectorQueryException(
"Unknown scope: {0}. Must be one of: {1}".format(repr(scope), ", ".join(self.SCOPES)))
self.scope = '_' + scope
self.db = DBHandle(globals()['__salt__']['config.get']('inspector.db', ''))
self.local_identity = dict()
def __init__(self, cachedir=None, piddir=None, pidfilename=None):
"""
Constructor.
:param options:
:param db_path:
:param pid_file:
"""
if not cachedir and "__salt__" in globals():
cachedir = globals().get("__salt__")["config.get"]("inspector.db", "")
self.dbfile = os.path.join(cachedir or self.DEFAULT_CACHE_PATH, self.DB_FILE)
self.db = DBHandle(self.dbfile)
if not piddir and "__salt__" in globals():
piddir = globals().get("__salt__")["config.get"]("inspector.pid", "")
self.pidfile = os.path.join(
piddir or self.DEFAULT_PID_PATH, pidfilename or self.PID_FILE
)
def __init__(self, cachedir=None, piddir=None, pidfilename=None):
'''
Constructor.
:param options:
:param db_path:
:param pid_file:
'''
if not cachedir and '__salt__' in globals():
cachedir = globals().get('__salt__')['config.get']('inspector.db',
'')
self.dbfile = os.path.join(cachedir or self.DEFAULT_CACHE_PATH,
self.DB_FILE)
self.db = DBHandle(self.dbfile)
if not piddir and '__salt__' in globals():
piddir = globals().get('__salt__')['config.get']('inspector.pid',
'')
self.pidfile = os.path.join(piddir or self.DEFAULT_PID_PATH,
pidfilename or self.PID_FILE)
class Inspector(object):
MODE = ['configuration', 'payload', 'all']
IGNORE_MOUNTS = [
"proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"
]
IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"]
IGNORE_PATHS = [
"/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm",
"/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home"
]
def __init__(self, db_path=None, pid_file=None):
# Configured path
if not db_path and '__salt__' in globals():
db_path = globals().get('__salt__')['config.get']('inspector.db',
'')
if not db_path:
raise InspectorSnapshotException(
"Inspector database location is not configured yet in minion.")
self.dbfile = db_path
self.db = DBHandle(self.dbfile)
self.db.open()
if not pid_file and '__salt__' in globals():
pid_file = globals().get('__salt__')['config.get']('inspector.pid',
'')
if not pid_file:
raise InspectorSnapshotException(
"Inspector PID file location is not configured yet in minion.")
self.pidfile = pid_file
def _syscall(self, command, input=None, env=None, *params):
'''
Call an external system command.
'''
return Popen([command] + list(params),
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
env=env or os.environ).communicate(input=input)
def _get_cfg_pkgs(self):
'''
Get packages with configuration files.
'''
out, err = self._syscall('rpm', None, None, '-qa', '--configfiles',
'--queryformat',
'%{name}-%{version}-%{release}\\n')
data = dict()
pkg_name = None
pkg_configs = []
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line:
continue
if not line.startswith("/"):
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
pkg_name = line
pkg_configs = []
else:
pkg_configs.append(line)
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
return data
def _get_changed_cfg_pkgs(self, data):
'''
Filter out unchanged packages.
'''
f_data = dict()
for pkg_name, pkg_files in data.items():
cfgs = list()
out, err = self._syscall("rpm", None, None, '-V', '--nodeps',
'--nodigest', '--nosignature',
'--nomtime', '--nolinkto', pkg_name)
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line or line.find(" c ") < 0 or line.split(" ")[0].find(
"5") < 0:
continue
cfg_file = line.split(" ")[-1]
if cfg_file in pkg_files:
cfgs.append(cfg_file)
if cfgs:
f_data[pkg_name] = cfgs
return f_data
def _save_cfg_pkgs(self, data):
'''
Save configuration packages.
'''
for table in ["inspector_pkg", "inspector_pkg_cfg_files"]:
self.db.flush(table)
pkg_id = 0
pkg_cfg_id = 0
for pkg_name, pkg_configs in data.items():
self.db.cursor.execute(
"INSERT INTO inspector_pkg (id, name) VALUES (?, ?)",
(pkg_id, pkg_name))
for pkg_config in pkg_configs:
self.db.cursor.execute(
"INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)",
(pkg_cfg_id, pkg_id, pkg_config))
pkg_cfg_id += 1
pkg_id += 1
self.db.connection.commit()
def _save_payload(self, files, directories, links):
'''
Save payload (unmanaged files)
'''
idx = 0
for p_type, p_list in (
('f', files),
('d', directories),
(
'l',
links,
),
):
for p_obj in p_list:
stats = os.stat(p_obj)
self.db.cursor.execute(
"INSERT INTO inspector_payload "
"(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)"
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(idx, p_obj, p_type, stats.st_mode, stats.st_uid,
stats.st_gid, stats.st_size, stats.st_atime,
stats.st_mtime, stats.st_ctime))
idx += 1
self.db.connection.commit()
def _get_managed_files(self):
'''
Build a in-memory data of all managed files.
'''
dirs = set()
links = set()
files = set()
cmd = __salt__['cmd.run_stdout']('rpm -qlav')
for line in cmd:
line = line.strip()
if not line:
continue
line = line.replace("\t", " ").split(" ")
if line[0][0] == "d":
dirs.add(line[-1])
elif line[0][0] == "l":
links.add(line[-1])
elif line[0][0] == "-":
files.add(line[-1])
return sorted(files), sorted(dirs), sorted(links)
def _get_all_files(self, path, *exclude):
'''
Walk implementation. Version in python 2.x and 3.x works differently.
'''
files = list()
dirs = list()
links = list()
for obj in os.listdir(path):
obj = os.path.join(path, obj)
valid = True
for ex_obj in exclude:
if obj.startswith(str(ex_obj)):
valid = False
continue
if not valid or not os.path.exists(obj):
continue
mode = os.lstat(obj).st_mode
if stat.S_ISLNK(mode):
links.append(obj)
elif stat.S_ISDIR(mode):
dirs.append(obj)
f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude)
files.extend(f_obj)
dirs.extend(d_obj)
links.extend(l_obj)
elif stat.S_ISREG(mode):
files.append(obj)
return sorted(files), sorted(dirs), sorted(links)
def _get_unmanaged_files(self, managed, system_all):
'''
Get the intersection between all files and managed files.
'''
def intr(src, data):
out = set()
for d_el in data:
if d_el not in src:
out.add(d_el)
return out
m_files, m_dirs, m_links = managed
s_files, s_dirs, s_links = system_all
return sorted(intr(m_files,
s_files)), sorted(intr(m_dirs, s_dirs)), sorted(
intr(m_links, s_links))
def _scan_payload(self):
'''
Scan the system.
'''
# Get ignored points
allowed = list()
self.db.cursor.execute("SELECT path FROM inspector_allowed")
for alwd_path in self.db.cursor.fetchall():
if os.path.exists(alwd_path[0]):
allowed.append(alwd_path[0])
ignored = list()
if not allowed:
self.db.cursor.execute("SELECT path FROM inspector_ignored")
for ign_path in self.db.cursor.fetchall():
ignored.append(ign_path[0])
all_files = list()
all_dirs = list()
all_links = list()
for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]:
if entry_path[0] != "/":
entry_path = "/{0}".format(entry_path)
if entry_path in ignored:
continue
e_files, e_dirs, e_links = self._get_all_files(
entry_path, *ignored)
all_files.extend(e_files)
all_dirs.extend(e_dirs)
all_links.extend(e_links)
return self._get_unmanaged_files(self._get_managed_files(), (
all_files,
all_dirs,
all_links,
))
def _prepare_full_scan(self, **kwargs):
'''
Prepare full system scan by setting up the database etc.
'''
# TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried.
self.db.purge()
# Add ignored filesystems
ignored_fs = set()
ignored_fs |= set(self.IGNORE_PATHS)
mounts = fsutils._get_mounts()
for device, data in mounts.items():
if device in self.IGNORE_MOUNTS:
for mpt in data:
ignored_fs.add(mpt['mount_point'])
continue
for mpt in data:
if mpt['type'] in self.IGNORE_FS_TYPES:
ignored_fs.add(mpt['mount_point'])
# Remove leafs of ignored filesystems
ignored_all = list()
for entry in sorted(list(ignored_fs)):
valid = True
for e_entry in ignored_all:
if entry.startswith(e_entry):
valid = False
break
if valid:
ignored_all.append(entry)
# Save to the database for further scan
for ignored_dir in ignored_all:
self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)",
(ignored_dir, ))
# Add allowed filesystems (overrides all above at full scan)
allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm]
for allowed_dir in allowed:
self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)",
(allowed_dir, ))
self.db.connection.commit()
return ignored_all
def snapshot(self, mode):
'''
Take a snapshot of the system.
'''
# TODO: Mode
self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs()))
self._save_payload(*self._scan_payload())
def request_snapshot(self, mode, priority=19, **kwargs):
'''
Take a snapshot of the system.
'''
if mode not in self.MODE:
raise InspectorSnapshotException(
"Unknown mode: '{0}'".format(mode))
self._prepare_full_scan(**kwargs)
os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format(
priority, __file__, self.pidfile, self.dbfile, mode))
class Inspector(object):
DEFAULT_MINION_CONFIG_PATH = '/etc/salt/minion'
MODE = ['configuration', 'payload', 'all']
IGNORE_MOUNTS = [
"proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"
]
IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"]
IGNORE_PATHS = [
"/tmp", "/var/tmp", "/lost+found", "/var/run", "/var/lib/rpm",
"/.snapshots", "/.zfs", "/etc/ssh", "/root", "/home"
]
def __init__(self, db_path=None, pid_file=None):
# Configured path
if not db_path and '__salt__' in globals():
db_path = globals().get('__salt__')['config.get']('inspector.db',
'')
if not db_path:
raise InspectorSnapshotException(
'Inspector database location is not configured yet in minion.\n'
'Add "inspector.db: /path/to/cache" in "/etc/salt/minion".')
self.dbfile = db_path
self.db = DBHandle(self.dbfile)
self.db.open()
if not pid_file and '__salt__' in globals():
pid_file = globals().get('__salt__')['config.get']('inspector.pid',
'')
if not pid_file:
raise InspectorSnapshotException(
"Inspector PID file location is not configured yet in minion.\n"
'Add "inspector.pid: /path/to/pids in "/etc/salt/minion".')
self.pidfile = pid_file
def _syscall(self, command, input=None, env=None, *params):
'''
Call an external system command.
'''
return Popen([command] + list(params),
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
env=env or os.environ).communicate(input=input)
def _get_cfg_pkgs(self):
'''
Package scanner switcher between the platforms.
:return:
'''
if self.grains_core.os_data().get('os_family') == 'Debian':
return self.__get_cfg_pkgs_dpkg()
elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']:
return self.__get_cfg_pkgs_rpm()
else:
return dict()
def __get_cfg_pkgs_dpkg(self):
'''
Get packages with configuration files on Dpkg systems.
:return:
'''
# Get list of all available packages
data = dict()
for pkg_name in salt.utils.to_str(
self._syscall('dpkg-query', None, None, '-Wf',
"${binary:Package}\\n")[0]).split(os.linesep):
pkg_name = pkg_name.strip()
if not pkg_name:
continue
data[pkg_name] = list()
for pkg_cfg_item in salt.utils.to_str(
self._syscall('dpkg-query', None, None, '-Wf',
"${Conffiles}\\n",
pkg_name)[0]).split(os.linesep):
pkg_cfg_item = pkg_cfg_item.strip()
if not pkg_cfg_item:
continue
pkg_cfg_file, pkg_cfg_sum = pkg_cfg_item.strip().split(" ", 1)
data[pkg_name].append(pkg_cfg_file)
# Dpkg meta data is unreliable. Check every package
# and remove which actually does not have config files.
if not data[pkg_name]:
data.pop(pkg_name)
return data
def __get_cfg_pkgs_rpm(self):
'''
Get packages with configuration files on RPM systems.
'''
out, err = self._syscall('rpm', None, None, '-qa', '--configfiles',
'--queryformat',
'%{name}-%{version}-%{release}\\n')
data = dict()
pkg_name = None
pkg_configs = []
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line:
continue
if not line.startswith("/"):
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
pkg_name = line
pkg_configs = []
else:
pkg_configs.append(line)
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
return data
def _get_changed_cfg_pkgs(self, data):
'''
Filter out unchanged packages on the Debian or RPM systems.
:param data: Structure {package-name -> [ file .. file1 ]}
:return: Same structure as data, except only files that were changed.
'''
f_data = dict()
for pkg_name, pkg_files in data.items():
cfgs = list()
cfg_data = list()
if self.grains_core.os_data().get('os_family') == 'Debian':
cfg_data = salt.utils.to_str(
self._syscall("dpkg", None, None, '--verify',
pkg_name)[0]).split(os.linesep)
elif self.grains_core.os_data().get('os_family') in [
'Suse', 'redhat'
]:
cfg_data = salt.utils.to_str(
self._syscall("rpm", None, None, '-V', '--nodeps',
'--nodigest', '--nosignature', '--nomtime',
'--nolinkto', pkg_name)[0]).split(os.linesep)
for line in cfg_data:
line = line.strip()
if not line or line.find(" c ") < 0 or line.split(" ")[0].find(
"5") < 0:
continue
cfg_file = line.split(" ")[-1]
if cfg_file in pkg_files:
cfgs.append(cfg_file)
if cfgs:
f_data[pkg_name] = cfgs
return f_data
def _save_cfg_pkgs(self, data):
'''
Save configuration packages.
'''
for table in ["inspector_pkg", "inspector_pkg_cfg_files"]:
self.db.flush(table)
pkg_id = 0
pkg_cfg_id = 0
for pkg_name, pkg_configs in data.items():
self.db.cursor.execute(
"INSERT INTO inspector_pkg (id, name) VALUES (?, ?)",
(pkg_id, pkg_name))
for pkg_config in pkg_configs:
self.db.cursor.execute(
"INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)",
(pkg_cfg_id, pkg_id, pkg_config))
pkg_cfg_id += 1
pkg_id += 1
self.db.connection.commit()
def _save_payload(self, files, directories, links):
'''
Save payload (unmanaged files)
'''
idx = 0
for p_type, p_list in (
('f', files),
('d', directories),
(
'l',
links,
),
):
for p_obj in p_list:
stats = os.stat(p_obj)
self.db.cursor.execute(
"INSERT INTO inspector_payload "
"(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)"
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(idx, p_obj, p_type, stats.st_mode, stats.st_uid,
stats.st_gid, stats.st_size, stats.st_atime,
stats.st_mtime, stats.st_ctime))
idx += 1
self.db.connection.commit()
def _get_managed_files(self):
'''
Build a in-memory data of all managed files.
'''
if self.grains_core.os_data().get('os_family') == 'Debian':
return self.__get_managed_files_dpkg()
elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']:
return self.__get_managed_files_rpm()
return list(), list(), list()
def __get_managed_files_dpkg(self):
'''
Get a list of all system files, belonging to the Debian package manager.
'''
dirs = set()
links = set()
files = set()
for pkg_name in salt.utils.to_str(
self._syscall("dpkg-query", None, None, '-Wf',
'${binary:Package}\\n')[0]).split(os.linesep):
pkg_name = pkg_name.strip()
if not pkg_name:
continue
for resource in salt.utils.to_str(
self._syscall("dpkg", None, None, '-L',
pkg_name)[0]).split(os.linesep):
resource = resource.strip()
if not resource or resource in ['/', './', '.']:
continue
if os.path.isdir(resource):
dirs.add(resource)
elif os.path.islink(resource):
links.add(resource)
elif os.path.isfile(resource):
files.add(resource)
return sorted(files), sorted(dirs), sorted(links)
def __get_managed_files_rpm(self):
'''
Get a list of all system files, belonging to the RedHat package manager.
'''
dirs = set()
links = set()
files = set()
for line in salt.utils.to_str(
self._syscall("rpm", None, None,
'-qlav')[0]).split(os.linesep):
line = line.strip()
if not line:
continue
line = line.replace("\t", " ").split(" ")
if line[0][0] == "d":
dirs.add(line[-1])
elif line[0][0] == "l":
links.add(line[-1])
elif line[0][0] == "-":
files.add(line[-1])
return sorted(files), sorted(dirs), sorted(links)
def _get_all_files(self, path, *exclude):
'''
Walk implementation. Version in python 2.x and 3.x works differently.
'''
files = list()
dirs = list()
links = list()
for obj in os.listdir(path):
obj = os.path.join(path, obj)
valid = True
for ex_obj in exclude:
if obj.startswith(str(ex_obj)):
valid = False
continue
if not valid or not os.path.exists(obj):
continue
if os.path.islink(obj):
links.append(obj)
elif os.path.isdir(obj):
dirs.append(obj)
f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude)
files.extend(f_obj)
dirs.extend(d_obj)
links.extend(l_obj)
elif os.path.isfile(obj):
files.append(obj)
return sorted(files), sorted(dirs), sorted(links)
def _get_unmanaged_files(self, managed, system_all):
'''
Get the intersection between all files and managed files.
'''
def intr(src, data):
out = set()
for d_el in data:
if d_el not in src:
out.add(d_el)
return out
m_files, m_dirs, m_links = managed
s_files, s_dirs, s_links = system_all
return sorted(intr(m_files,
s_files)), sorted(intr(m_dirs, s_dirs)), sorted(
intr(m_links, s_links))
def _scan_payload(self):
'''
Scan the system.
'''
# Get ignored points
allowed = list()
self.db.cursor.execute("SELECT path FROM inspector_allowed")
for alwd_path in self.db.cursor.fetchall():
if os.path.exists(alwd_path[0]):
allowed.append(alwd_path[0])
ignored = list()
if not allowed:
self.db.cursor.execute("SELECT path FROM inspector_ignored")
for ign_path in self.db.cursor.fetchall():
ignored.append(ign_path[0])
all_files = list()
all_dirs = list()
all_links = list()
for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]:
if entry_path[0] != "/":
entry_path = "/{0}".format(entry_path)
if entry_path in ignored or os.path.islink(entry_path):
continue
e_files, e_dirs, e_links = self._get_all_files(
entry_path, *ignored)
all_files.extend(e_files)
all_dirs.extend(e_dirs)
all_links.extend(e_links)
return self._get_unmanaged_files(self._get_managed_files(), (
all_files,
all_dirs,
all_links,
))
def _prepare_full_scan(self, **kwargs):
'''
Prepare full system scan by setting up the database etc.
'''
# TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried.
self.db.purge()
# Add ignored filesystems
ignored_fs = set()
ignored_fs |= set(self.IGNORE_PATHS)
mounts = fsutils._get_mounts()
for device, data in mounts.items():
if device in self.IGNORE_MOUNTS:
for mpt in data:
ignored_fs.add(mpt['mount_point'])
continue
for mpt in data:
if mpt['type'] in self.IGNORE_FS_TYPES:
ignored_fs.add(mpt['mount_point'])
# Remove leafs of ignored filesystems
ignored_all = list()
for entry in sorted(list(ignored_fs)):
valid = True
for e_entry in ignored_all:
if entry.startswith(e_entry):
valid = False
break
if valid:
ignored_all.append(entry)
# Save to the database for further scan
for ignored_dir in ignored_all:
self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)",
(ignored_dir, ))
# Add allowed filesystems (overrides all above at full scan)
allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm]
for allowed_dir in allowed:
self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)",
(allowed_dir, ))
self.db.connection.commit()
return ignored_all
def _init_env(self):
'''
Initialize some Salt environment.
'''
from salt.config import minion_config
from salt.grains import core as g_core
g_core.__opts__ = minion_config(self.DEFAULT_MINION_CONFIG_PATH)
self.grains_core = g_core
def snapshot(self, mode):
'''
Take a snapshot of the system.
'''
self._init_env()
self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs()))
self._save_payload(*self._scan_payload())
def request_snapshot(self, mode, priority=19, **kwargs):
'''
Take a snapshot of the system.
'''
if mode not in self.MODE:
raise InspectorSnapshotException(
"Unknown mode: '{0}'".format(mode))
self._prepare_full_scan(**kwargs)
os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format(
priority, __file__, self.pidfile, self.dbfile, mode))
class Query(object):
'''
Query the system.
This class is actually puts all Salt features together,
so there would be no need to pick it from various places.
'''
# Configuration: config files
# Identity: users/groups
# Software: packages, patterns, repositories
# Services
# System: distro, RAM etc
# Changes: all files that are managed and were changed from the original
# all: include all scopes (scary!)
# payload: files that are not managed
SCOPES = ["changes", "configuration", "identity", "system", "software", "services", "payload", "all"]
def __init__(self, scope):
'''
Constructor.
:param scope:
:return:
'''
if scope not in self.SCOPES:
raise InspectorQueryException(
"Unknown scope: {0}. Must be one of: {1}".format(repr(scope), ", ".join(self.SCOPES)))
self.scope = '_' + scope
self.db = DBHandle(globals()['__salt__']['config.get']('inspector.db', ''))
self.local_identity = dict()
def __call__(self, *args, **kwargs):
'''
Call the query with the defined scope.
:param args:
:param kwargs:
:return:
'''
return getattr(self, self.scope)(*args, **kwargs)
def _changes(self, *args, **kwargs):
'''
Returns all diffs to the configuration files.
'''
raise Exception("Not yet implemented")
def _configuration(self, *args, **kwargs):
'''
Return configuration files.
'''
data = dict()
self.db.open()
self.db.cursor.execute("SELECT id, name FROM inspector_pkg")
for pkg_id, pkg_name in self.db.cursor.fetchall():
self.db.cursor.execute("SELECT id, path FROM inspector_pkg_cfg_files WHERE pkgid=?", (pkg_id,))
configs = list()
for cnf_id, cnf_name in self.db.cursor.fetchall():
configs.append(cnf_name)
data[pkg_name] = configs
self.db.close()
if not data:
raise InspectorQueryException("No inspected configuration yet available.")
return data
def _get_local_users(self, disabled=None):
'''
Return all known local accounts to the system.
'''
users = dict()
path = '/etc/passwd'
with salt.utils.fopen(path, 'r') as fp_:
for line in fp_:
line = line.strip()
if ':' not in line:
continue
name, password, uid, gid, gecos, directory, shell = line.split(':')
active = not (password == '*' or password.startswith('!'))
if (disabled is False and active) or (disabled is True and not active) or disabled is None:
users[name] = {
'uid': uid,
'git': gid,
'info': gecos,
'home': directory,
'shell': shell,
'disabled': not active
}
return users
def _get_local_groups(self):
'''
Return all known local groups to the system.
'''
groups = dict()
path = '/etc/group'
with salt.utils.fopen(path, 'r') as fp_:
for line in fp_:
line = line.strip()
if ':' not in line:
continue
name, password, gid, users = line.split(':')
groups[name] = {
'gid': gid,
}
if users:
groups[name]['users'] = users.split(',')
return groups
def _get_external_accounts(self, locals):
'''
Return all known accounts, excluding local accounts.
'''
users = dict()
out = __salt__['cmd.run_all']("passwd -S -a")
if out['retcode']:
# System does not supports all accounts descriptions, just skipping.
return users
status = {'L': 'Locked', 'NP': 'No password', 'P': 'Usable password', 'LK': 'Locked'}
for data in [elm.strip().split(" ") for elm in out['stdout'].split(os.linesep) if elm.strip()]:
if len(data) < 2:
continue
name, login = data[:2]
if name not in locals:
users[name] = {
'login': login,
'status': status.get(login, 'N/A')
}
return users
def _identity(self, *args, **kwargs):
'''
Local users and groups.
accounts
Can be either 'local', 'remote' or 'all' (equal to "local,remote").
Remote accounts cannot be resolved on all systems, but only
those, which supports 'passwd -S -a'.
disabled
True (or False, default) to return only disabled accounts.
'''
LOCAL = 'local accounts'
EXT = 'external accounts'
data = dict()
data[LOCAL] = self._get_local_users(disabled=kwargs.get('disabled'))
data[EXT] = self._get_external_accounts(data[LOCAL].keys()) or 'N/A'
data['local groups'] = self._get_local_groups()
return data
def _system(self, *args, **kwargs):
'''
This basically calls grains items and picks out only
necessary information in a certain structure.
:param args:
:param kwargs:
:return:
'''
sysinfo = SysInfo(__grains__.get("kernel"))
data = dict()
data['cpu'] = sysinfo._get_cpu()
data['disks'] = sysinfo._get_fs()
data['mounts'] = sysinfo._get_mounts()
data['memory'] = sysinfo._get_mem()
data['network'] = sysinfo._get_network()
data['os'] = sysinfo._get_os()
return data
def _software(self, *args, **kwargs):
'''
Return installed software.
'''
data = dict()
if 'exclude' in kwargs:
excludes = kwargs['exclude'].split(",")
else:
excludes = list()
os_family = __grains__.get("os_family").lower()
# Get locks
if os_family == 'suse':
LOCKS = "pkg.list_locks"
if 'products' not in excludes:
products = __salt__['pkg.list_products']()
if products:
data['products'] = products
elif os_family == 'redhat':
LOCKS = "pkg.get_locked_packages"
else:
LOCKS = None
if LOCKS and 'locks' not in excludes:
locks = __salt__[LOCKS]()
if locks:
data['locks'] = locks
# Get patterns
if os_family == 'suse':
PATTERNS = 'pkg.list_installed_patterns'
elif os_family == 'redhat':
PATTERNS = 'pkg.group_list'
else:
PATTERNS = None
if PATTERNS and 'patterns' not in excludes:
patterns = __salt__[PATTERNS]()
if patterns:
data['patterns'] = patterns
# Get packages
if 'packages' not in excludes:
data['packages'] = __salt__['pkg.list_pkgs']()
# Get repositories
if 'repositories' not in excludes:
repos = __salt__['pkg.list_repos']()
if repos:
data['repositories'] = repos
return data
def _services(self, *args, **kwargs):
'''
Get list of enabled and disabled services on the particular system.
'''
return {
'enabled': __salt__['service.get_enabled'](),
'disabled': __salt__['service.get_disabled'](),
}
def _id_resolv(self, iid, named=True, uid=True):
'''
Resolve local users and groups.
:param iid:
:param named:
:param uid:
:return:
'''
if not self.local_identity:
self.local_identity['users'] = self._get_local_users()
self.local_identity['groups'] = self._get_local_groups()
if not named:
return iid
for name, meta in self.local_identity[uid and 'users' or 'groups'].items():
if (uid and int(meta.get('uid', -1)) == iid) or (not uid and int(meta.get('gid', -1)) == iid):
return name
return iid
def _payload(self, *args, **kwargs):
'''
Find all unmanaged files.
Parameters:
* **filter**: Include only results which path starts from the filter string.
* **time**: Display time in Unix ticks or format according to the configured TZ (default)
Values: ticks, tz (default)
* **size**: Format size. Values: B, KB, MB, GB
* **owners**: Resolve UID/GID to an actual names or leave them numeric (default).
Values: name (default), id
* **type**: Comma-separated type of included payload: dir (or directory), link and/or file.
* **brief**: Return just a list of matches, if True. Default: False
'''
def _size_format(size, fmt):
if fmt is None:
return size
fmt = fmt.lower()
if fmt == "b":
return "{0} Bytes".format(size)
elif fmt == "kb":
return "{0} Kb".format(round((float(size) / 0x400), 2))
elif fmt == "mb":
return "{0} Mb".format(round((float(size) / 0x400 / 0x400), 2))
elif fmt == "gb":
return "{0} Gb".format(round((float(size) / 0x400 / 0x400 / 0x400), 2))
filter = None
if 'filter' in kwargs:
filter = kwargs['filter']
timeformat = kwargs.get("time", "tz")
if timeformat not in ["ticks", "tz"]:
raise InspectorQueryException('Unknown "{0}" value for parameter "time"'.format(timeformat))
tfmt = lambda param: timeformat == "tz" and time.strftime("%b %d %Y %H:%M:%S", time.gmtime(param)) or int(param)
size_fmt = kwargs.get("size")
if size_fmt is not None and size_fmt.lower() not in ["b", "kb", "mb", "gb"]:
raise InspectorQueryException('Unknown "{0}" value for parameter "size". '
'Should be either B, Kb, Mb or Gb'.format(timeformat))
owners = kwargs.get("owners", "id")
if owners not in ["name", "id"]:
raise InspectorQueryException('Unknown "{0}" value for parameter "owners". '
'Should be either name or id (default)'.format(owners))
incl_type = [prm for prm in kwargs.get("type", "").lower().split(",") if prm]
if not incl_type:
incl_type.append("file")
for i_type in incl_type:
if i_type not in ["directory", "dir", "d", "file", "f", "link", "l"]:
raise InspectorQueryException('Unknown "{0}" values for parameter "type". '
'Should be comma separated one or more of '
'dir, file and/or link.'.format(", ".join(incl_type)))
where_clause = set()
for i_type in incl_type:
if i_type in ["file", "f"]:
where_clause.add("p_type = 'f'")
elif i_type in ["d", "dir", "directory"]:
where_clause.add("p_type = 'd'")
elif i_type in ["l", "link"]:
where_clause.add("p_type = 'l'")
if filter:
where_filter_clause = " AND path LIKE '{0}%'".format(filter)
else:
where_filter_clause = ""
self.db.open()
self.db.cursor.execute("SELECT id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime "
"FROM inspector_payload "
"WHERE {0}{1}".format(" OR ".join(list(where_clause)),
where_filter_clause))
brief = kwargs.get("brief")
if brief:
data = list()
else:
data = dict()
for pld_data in self.db.cursor.fetchall():
p_id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime = pld_data
if brief:
data.append(path)
else:
data[path] = {
'uid': self._id_resolv(uid, named=(owners == "id")),
'gid': self._id_resolv(gid, named=(owners == "id"), uid=False),
'size': _size_format(p_size, fmt=size_fmt),
'mode': oct(mode),
'accessed': tfmt(atime),
'modified': tfmt(mtime),
'created': tfmt(ctime),
}
self.db.close()
return data
def _all(self, *args, **kwargs):
'''
Return all the summary of the particular system.
'''
data = dict()
data['software'] = self._software(**kwargs)
data['system'] = self._system(**kwargs)
data['services'] = self._services(**kwargs)
try:
data['configuration'] = self._configuration(**kwargs)
except InspectorQueryException as ex:
data['configuration'] = 'N/A'
log.error(ex)
data['payload'] = self._payload(**kwargs) or 'N/A'
return data
class Inspector(object):
DEFAULT_MINION_CONFIG_PATH = '/etc/salt/minion'
MODE = ['configuration', 'payload', 'all']
IGNORE_MOUNTS = ["proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"]
IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"]
IGNORE_PATHS = ["/tmp", "/var/tmp", "/lost+found", "/var/run",
"/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh",
"/root", "/home"]
def __init__(self, db_path=None, pid_file=None):
# Configured path
if not db_path and '__salt__' in globals():
db_path = globals().get('__salt__')['config.get']('inspector.db', '')
if not db_path:
raise InspectorSnapshotException('Inspector database location is not configured yet in minion.\n'
'Add "inspector.db: /path/to/cache" in "/etc/salt/minion".')
self.dbfile = db_path
self.db = DBHandle(self.dbfile)
self.db.open()
if not pid_file and '__salt__' in globals():
pid_file = globals().get('__salt__')['config.get']('inspector.pid', '')
if not pid_file:
raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.\n"
'Add "inspector.pid: /path/to/pids in "/etc/salt/minion".')
self.pidfile = pid_file
def _syscall(self, command, input=None, env=None, *params):
'''
Call an external system command.
'''
return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT,
env=env or os.environ).communicate(input=input)
def _get_cfg_pkgs(self):
'''
Package scanner switcher between the platforms.
:return:
'''
if self.grains_core.os_data().get('os_family') == 'Debian':
return self.__get_cfg_pkgs_dpkg()
elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']:
return self.__get_cfg_pkgs_rpm()
else:
return dict()
def __get_cfg_pkgs_dpkg(self):
'''
Get packages with configuration files on Dpkg systems.
:return:
'''
# Get list of all available packages
data = dict()
for pkg_name in salt.utils.to_str(self._syscall('dpkg-query', None, None,
'-Wf', "${binary:Package}\\n")[0]).split(os.linesep):
pkg_name = pkg_name.strip()
if not pkg_name:
continue
data[pkg_name] = list()
for pkg_cfg_item in salt.utils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${Conffiles}\\n",
pkg_name)[0]).split(os.linesep):
pkg_cfg_item = pkg_cfg_item.strip()
if not pkg_cfg_item:
continue
pkg_cfg_file, pkg_cfg_sum = pkg_cfg_item.strip().split(" ", 1)
data[pkg_name].append(pkg_cfg_file)
# Dpkg meta data is unreliable. Check every package
# and remove which actually does not have config files.
if not data[pkg_name]:
data.pop(pkg_name)
return data
def __get_cfg_pkgs_rpm(self):
'''
Get packages with configuration files on RPM systems.
'''
out, err = self._syscall('rpm', None, None, '-qa', '--configfiles',
'--queryformat', '%{name}-%{version}-%{release}\\n')
data = dict()
pkg_name = None
pkg_configs = []
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line:
continue
if not line.startswith("/"):
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
pkg_name = line
pkg_configs = []
else:
pkg_configs.append(line)
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
return data
def _get_changed_cfg_pkgs(self, data):
'''
Filter out unchanged packages on the Debian or RPM systems.
:param data: Structure {package-name -> [ file .. file1 ]}
:return: Same structure as data, except only files that were changed.
'''
f_data = dict()
for pkg_name, pkg_files in data.items():
cfgs = list()
cfg_data = list()
if self.grains_core.os_data().get('os_family') == 'Debian':
cfg_data = salt.utils.to_str(self._syscall("dpkg", None, None, '--verify',
pkg_name)[0]).split(os.linesep)
elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']:
cfg_data = salt.utils.to_str(self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest',
'--nosignature', '--nomtime', '--nolinkto',
pkg_name)[0]).split(os.linesep)
for line in cfg_data:
line = line.strip()
if not line or line.find(" c ") < 0 or line.split(" ")[0].find("5") < 0:
continue
cfg_file = line.split(" ")[-1]
if cfg_file in pkg_files:
cfgs.append(cfg_file)
if cfgs:
f_data[pkg_name] = cfgs
return f_data
def _save_cfg_pkgs(self, data):
'''
Save configuration packages.
'''
for table in ["inspector_pkg", "inspector_pkg_cfg_files"]:
self.db.flush(table)
pkg_id = 0
pkg_cfg_id = 0
for pkg_name, pkg_configs in data.items():
self.db.cursor.execute("INSERT INTO inspector_pkg (id, name) VALUES (?, ?)",
(pkg_id, pkg_name))
for pkg_config in pkg_configs:
self.db.cursor.execute("INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)",
(pkg_cfg_id, pkg_id, pkg_config))
pkg_cfg_id += 1
pkg_id += 1
self.db.connection.commit()
def _save_payload(self, files, directories, links):
'''
Save payload (unmanaged files)
'''
idx = 0
for p_type, p_list in (('f', files), ('d', directories), ('l', links,),):
for p_obj in p_list:
stats = os.stat(p_obj)
self.db.cursor.execute("INSERT INTO inspector_payload "
"(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)"
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size,
stats.st_atime, stats.st_mtime, stats.st_ctime))
idx += 1
self.db.connection.commit()
def _get_managed_files(self):
'''
Build a in-memory data of all managed files.
'''
if self.grains_core.os_data().get('os_family') == 'Debian':
return self.__get_managed_files_dpkg()
elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']:
return self.__get_managed_files_rpm()
return list(), list(), list()
def __get_managed_files_dpkg(self):
'''
Get a list of all system files, belonging to the Debian package manager.
'''
dirs = set()
links = set()
files = set()
for pkg_name in salt.utils.to_str(self._syscall("dpkg-query", None, None,
'-Wf', '${binary:Package}\\n')[0]).split(os.linesep):
pkg_name = pkg_name.strip()
if not pkg_name:
continue
for resource in salt.utils.to_str(self._syscall("dpkg", None, None, '-L', pkg_name)[0]).split(os.linesep):
resource = resource.strip()
if not resource or resource in ['/', './', '.']:
continue
if os.path.isdir(resource):
dirs.add(resource)
elif os.path.islink(resource):
links.add(resource)
elif os.path.isfile(resource):
files.add(resource)
return sorted(files), sorted(dirs), sorted(links)
def __get_managed_files_rpm(self):
'''
Get a list of all system files, belonging to the RedHat package manager.
'''
dirs = set()
links = set()
files = set()
for line in salt.utils.to_str(self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep):
line = line.strip()
if not line:
continue
line = line.replace("\t", " ").split(" ")
if line[0][0] == "d":
dirs.add(line[-1])
elif line[0][0] == "l":
links.add(line[-1])
elif line[0][0] == "-":
files.add(line[-1])
return sorted(files), sorted(dirs), sorted(links)
def _get_all_files(self, path, *exclude):
'''
Walk implementation. Version in python 2.x and 3.x works differently.
'''
files = list()
dirs = list()
links = list()
for obj in os.listdir(path):
obj = os.path.join(path, obj)
valid = True
for ex_obj in exclude:
if obj.startswith(str(ex_obj)):
valid = False
continue
if not valid or not os.path.exists(obj):
continue
if os.path.islink(obj):
links.append(obj)
elif os.path.isdir(obj):
dirs.append(obj)
f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude)
files.extend(f_obj)
dirs.extend(d_obj)
links.extend(l_obj)
elif os.path.isfile(obj):
files.append(obj)
return sorted(files), sorted(dirs), sorted(links)
def _get_unmanaged_files(self, managed, system_all):
'''
Get the intersection between all files and managed files.
'''
def intr(src, data):
out = set()
for d_el in data:
if d_el not in src:
out.add(d_el)
return out
m_files, m_dirs, m_links = managed
s_files, s_dirs, s_links = system_all
return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted(intr(m_links, s_links))
def _scan_payload(self):
'''
Scan the system.
'''
# Get ignored points
allowed = list()
self.db.cursor.execute("SELECT path FROM inspector_allowed")
for alwd_path in self.db.cursor.fetchall():
if os.path.exists(alwd_path[0]):
allowed.append(alwd_path[0])
ignored = list()
if not allowed:
self.db.cursor.execute("SELECT path FROM inspector_ignored")
for ign_path in self.db.cursor.fetchall():
ignored.append(ign_path[0])
all_files = list()
all_dirs = list()
all_links = list()
for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]:
if entry_path[0] != "/":
entry_path = "/{0}".format(entry_path)
if entry_path in ignored or os.path.islink(entry_path):
continue
e_files, e_dirs, e_links = self._get_all_files(entry_path, *ignored)
all_files.extend(e_files)
all_dirs.extend(e_dirs)
all_links.extend(e_links)
return self._get_unmanaged_files(self._get_managed_files(), (all_files, all_dirs, all_links,))
def _prepare_full_scan(self, **kwargs):
'''
Prepare full system scan by setting up the database etc.
'''
# TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried.
self.db.purge()
# Add ignored filesystems
ignored_fs = set()
ignored_fs |= set(self.IGNORE_PATHS)
mounts = fsutils._get_mounts()
for device, data in mounts.items():
if device in self.IGNORE_MOUNTS:
for mpt in data:
ignored_fs.add(mpt['mount_point'])
continue
for mpt in data:
if mpt['type'] in self.IGNORE_FS_TYPES:
ignored_fs.add(mpt['mount_point'])
# Remove leafs of ignored filesystems
ignored_all = list()
for entry in sorted(list(ignored_fs)):
valid = True
for e_entry in ignored_all:
if entry.startswith(e_entry):
valid = False
break
if valid:
ignored_all.append(entry)
# Save to the database for further scan
for ignored_dir in ignored_all:
self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir,))
# Add allowed filesystems (overrides all above at full scan)
allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm]
for allowed_dir in allowed:
self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir,))
self.db.connection.commit()
return ignored_all
def _init_env(self):
'''
Initialize some Salt environment.
'''
from salt.config import minion_config
from salt.grains import core as g_core
g_core.__opts__ = minion_config(self.DEFAULT_MINION_CONFIG_PATH)
self.grains_core = g_core
def snapshot(self, mode):
'''
Take a snapshot of the system.
'''
self._init_env()
self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs()))
self._save_payload(*self._scan_payload())
def request_snapshot(self, mode, priority=19, **kwargs):
'''
Take a snapshot of the system.
'''
if mode not in self.MODE:
raise InspectorSnapshotException("Unknown mode: '{0}'".format(mode))
self._prepare_full_scan(**kwargs)
os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format(
priority, __file__, self.pidfile, self.dbfile, mode))
class Inspector(object):
MODE = ['configuration', 'payload', 'all']
IGNORE_MOUNTS = ["proc", "sysfs", "devtmpfs", "tmpfs", "fuse.gvfs-fuse-daemon"]
IGNORE_FS_TYPES = ["autofs", "cifs", "nfs", "nfs4"]
IGNORE_PATHS = ["/tmp", "/var/tmp", "/lost+found", "/var/run",
"/var/lib/rpm", "/.snapshots", "/.zfs", "/etc/ssh",
"/root", "/home"]
def __init__(self, db_path=None, pid_file=None):
# Configured path
if not db_path and '__salt__' in globals():
db_path = globals().get('__salt__')['config.get']('inspector.db', '')
if not db_path:
raise InspectorSnapshotException("Inspector database location is not configured yet in minion.")
self.dbfile = db_path
self.db = DBHandle(self.dbfile)
self.db.open()
if not pid_file and '__salt__' in globals():
pid_file = globals().get('__salt__')['config.get']('inspector.pid', '')
if not pid_file:
raise InspectorSnapshotException("Inspector PID file location is not configured yet in minion.")
self.pidfile = pid_file
def _syscall(self, command, input=None, env=None, *params):
'''
Call an external system command.
'''
return Popen([command] + list(params), stdout=PIPE, stdin=PIPE, stderr=STDOUT,
env=env or os.environ).communicate(input=input)
def _get_cfg_pkgs(self):
'''
Get packages with configuration files.
'''
out, err = self._syscall('rpm', None, None, '-qa', '--configfiles',
'--queryformat', '%{name}-%{version}-%{release}\\n')
data = dict()
pkg_name = None
pkg_configs = []
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line:
continue
if not line.startswith("/"):
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
pkg_name = line
pkg_configs = []
else:
pkg_configs.append(line)
if pkg_name and pkg_configs:
data[pkg_name] = pkg_configs
return data
def _get_changed_cfg_pkgs(self, data):
'''
Filter out unchanged packages.
'''
f_data = dict()
for pkg_name, pkg_files in data.items():
cfgs = list()
out, err = self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest',
'--nosignature', '--nomtime', '--nolinkto', pkg_name)
out = salt.utils.to_str(out)
for line in out.split(os.linesep):
line = line.strip()
if not line or line.find(" c ") < 0 or line.split(" ")[0].find("5") < 0:
continue
cfg_file = line.split(" ")[-1]
if cfg_file in pkg_files:
cfgs.append(cfg_file)
if cfgs:
f_data[pkg_name] = cfgs
return f_data
def _save_cfg_pkgs(self, data):
'''
Save configuration packages.
'''
for table in ["inspector_pkg", "inspector_pkg_cfg_files"]:
self.db.flush(table)
pkg_id = 0
pkg_cfg_id = 0
for pkg_name, pkg_configs in data.items():
self.db.cursor.execute("INSERT INTO inspector_pkg (id, name) VALUES (?, ?)",
(pkg_id, pkg_name))
for pkg_config in pkg_configs:
self.db.cursor.execute("INSERT INTO inspector_pkg_cfg_files (id, pkgid, path) VALUES (?, ?, ?)",
(pkg_cfg_id, pkg_id, pkg_config))
pkg_cfg_id += 1
pkg_id += 1
self.db.connection.commit()
def _save_payload(self, files, directories, links):
'''
Save payload (unmanaged files)
'''
idx = 0
for p_type, p_list in (('f', files), ('d', directories), ('l', links,),):
for p_obj in p_list:
stats = os.stat(p_obj)
self.db.cursor.execute("INSERT INTO inspector_payload "
"(id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime)"
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(idx, p_obj, p_type, stats.st_mode, stats.st_uid, stats.st_gid, stats.st_size,
stats.st_atime, stats.st_mtime, stats.st_ctime))
idx += 1
self.db.connection.commit()
def _get_managed_files(self):
'''
Build a in-memory data of all managed files.
'''
dirs = set()
links = set()
files = set()
cmd = __salt__['cmd.run_stdout']('rpm -qlav')
for line in cmd:
line = line.strip()
if not line:
continue
line = line.replace("\t", " ").split(" ")
if line[0][0] == "d":
dirs.add(line[-1])
elif line[0][0] == "l":
links.add(line[-1])
elif line[0][0] == "-":
files.add(line[-1])
return sorted(files), sorted(dirs), sorted(links)
def _get_all_files(self, path, *exclude):
'''
Walk implementation. Version in python 2.x and 3.x works differently.
'''
files = list()
dirs = list()
links = list()
for obj in os.listdir(path):
obj = os.path.join(path, obj)
valid = True
for ex_obj in exclude:
if obj.startswith(str(ex_obj)):
valid = False
continue
if not valid or not os.path.exists(obj):
continue
mode = os.lstat(obj).st_mode
if stat.S_ISLNK(mode):
links.append(obj)
elif stat.S_ISDIR(mode):
dirs.append(obj)
f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude)
files.extend(f_obj)
dirs.extend(d_obj)
links.extend(l_obj)
elif stat.S_ISREG(mode):
files.append(obj)
return sorted(files), sorted(dirs), sorted(links)
def _get_unmanaged_files(self, managed, system_all):
'''
Get the intersection between all files and managed files.
'''
def intr(src, data):
out = set()
for d_el in data:
if d_el not in src:
out.add(d_el)
return out
m_files, m_dirs, m_links = managed
s_files, s_dirs, s_links = system_all
return sorted(intr(m_files, s_files)), sorted(intr(m_dirs, s_dirs)), sorted(intr(m_links, s_links))
def _scan_payload(self):
'''
Scan the system.
'''
# Get ignored points
allowed = list()
self.db.cursor.execute("SELECT path FROM inspector_allowed")
for alwd_path in self.db.cursor.fetchall():
if os.path.exists(alwd_path[0]):
allowed.append(alwd_path[0])
ignored = list()
if not allowed:
self.db.cursor.execute("SELECT path FROM inspector_ignored")
for ign_path in self.db.cursor.fetchall():
ignored.append(ign_path[0])
all_files = list()
all_dirs = list()
all_links = list()
for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]:
if entry_path[0] != "/":
entry_path = "/{0}".format(entry_path)
if entry_path in ignored:
continue
e_files, e_dirs, e_links = self._get_all_files(entry_path, *ignored)
all_files.extend(e_files)
all_dirs.extend(e_dirs)
all_links.extend(e_links)
return self._get_unmanaged_files(self._get_managed_files(), (all_files, all_dirs, all_links,))
def _prepare_full_scan(self, **kwargs):
'''
Prepare full system scan by setting up the database etc.
'''
# TODO: Backup the SQLite database. Backup should be restored automatically if current db failed while queried.
self.db.purge()
# Add ignored filesystems
ignored_fs = set()
ignored_fs |= set(self.IGNORE_PATHS)
mounts = fsutils._get_mounts()
for device, data in mounts.items():
if device in self.IGNORE_MOUNTS:
for mpt in data:
ignored_fs.add(mpt['mount_point'])
continue
for mpt in data:
if mpt['type'] in self.IGNORE_FS_TYPES:
ignored_fs.add(mpt['mount_point'])
# Remove leafs of ignored filesystems
ignored_all = list()
for entry in sorted(list(ignored_fs)):
valid = True
for e_entry in ignored_all:
if entry.startswith(e_entry):
valid = False
break
if valid:
ignored_all.append(entry)
# Save to the database for further scan
for ignored_dir in ignored_all:
self.db.cursor.execute("INSERT INTO inspector_ignored VALUES (?)", (ignored_dir,))
# Add allowed filesystems (overrides all above at full scan)
allowed = [elm for elm in kwargs.get("filter", "").split(",") if elm]
for allowed_dir in allowed:
self.db.cursor.execute("INSERT INTO inspector_allowed VALUES (?)", (allowed_dir,))
self.db.connection.commit()
return ignored_all
def snapshot(self, mode):
'''
Take a snapshot of the system.
'''
# TODO: Mode
self._save_cfg_pkgs(self._get_changed_cfg_pkgs(self._get_cfg_pkgs()))
self._save_payload(*self._scan_payload())
def request_snapshot(self, mode, priority=19, **kwargs):
'''
Take a snapshot of the system.
'''
if mode not in self.MODE:
raise InspectorSnapshotException("Unknown mode: '{0}'".format(mode))
self._prepare_full_scan(**kwargs)
os.system("nice -{0} python {1} {2} {3} {4} & > /dev/null".format(
priority, __file__, self.pidfile, self.dbfile, mode))
class Query(object):
'''
Query the system.
This class is actually puts all Salt features together,
so there would be no need to pick it from various places.
'''
# Configuration: config files
# Identity: users/groups
# Software: packages, patterns, repositories
# Services
# System: distro, RAM etc
# Changes: all files that are managed and were changed from the original
# all: include all scopes (scary!)
# payload: files that are not managed
SCOPES = [
"changes", "configuration", "identity", "system", "software",
"services", "payload", "all"
]
def __init__(self, scope):
'''
Constructor.
:param scope:
:return:
'''
if scope not in self.SCOPES:
raise InspectorQueryException(
"Unknown scope: {0}. Must be one of: {1}".format(
repr(scope), ", ".join(self.SCOPES)))
self.scope = '_' + scope
self.db = DBHandle(globals()['__salt__']['config.get']('inspector.db',
''))
self.local_identity = dict()
def __call__(self, *args, **kwargs):
'''
Call the query with the defined scope.
:param args:
:param kwargs:
:return:
'''
return getattr(self, self.scope)(*args, **kwargs)
def _changes(self, *args, **kwargs):
'''
Returns all diffs to the configuration files.
'''
raise Exception("Not yet implemented")
def _configuration(self, *args, **kwargs):
'''
Return configuration files.
'''
data = dict()
self.db.open()
self.db.cursor.execute("SELECT id, name FROM inspector_pkg")
for pkg_id, pkg_name in self.db.cursor.fetchall():
self.db.cursor.execute(
"SELECT id, path FROM inspector_pkg_cfg_files WHERE pkgid=?",
(pkg_id, ))
configs = list()
for cnf_id, cnf_name in self.db.cursor.fetchall():
configs.append(cnf_name)
data[pkg_name] = configs
self.db.close()
if not data:
raise InspectorQueryException(
"No inspected configuration yet available.")
return data
def _get_local_users(self, disabled=None):
'''
Return all known local accounts to the system.
'''
users = dict()
path = '/etc/passwd'
with salt.utils.fopen(path, 'r') as fp_:
for line in fp_:
line = line.strip()
if ':' not in line:
continue
name, password, uid, gid, gecos, directory, shell = line.split(
':')
active = not (password == '*' or password.startswith('!'))
if (disabled is False
and active) or (disabled is True
and not active) or disabled is None:
users[name] = {
'uid': uid,
'git': gid,
'info': gecos,
'home': directory,
'shell': shell,
'disabled': not active
}
return users
def _get_local_groups(self):
'''
Return all known local groups to the system.
'''
groups = dict()
path = '/etc/group'
with salt.utils.fopen(path, 'r') as fp_:
for line in fp_:
line = line.strip()
if ':' not in line:
continue
name, password, gid, users = line.split(':')
groups[name] = {
'gid': gid,
}
if users:
groups[name]['users'] = users.split(',')
return groups
def _get_external_accounts(self, locals):
'''
Return all known accounts, excluding local accounts.
'''
users = dict()
out = __salt__['cmd.run_all']("passwd -S -a")
if out['retcode']:
# System does not supports all accounts descriptions, just skipping.
return users
status = {
'L': 'Locked',
'NP': 'No password',
'P': 'Usable password',
'LK': 'Locked'
}
for data in [
elm.strip().split(" ")
for elm in out['stdout'].split(os.linesep) if elm.strip()
]:
if len(data) < 2:
continue
name, login = data[:2]
if name not in locals:
users[name] = {
'login': login,
'status': status.get(login, 'N/A')
}
return users
def _identity(self, *args, **kwargs):
'''
Local users and groups.
accounts
Can be either 'local', 'remote' or 'all' (equal to "local,remote").
Remote accounts cannot be resolved on all systems, but only
those, which supports 'passwd -S -a'.
disabled
True (or False, default) to return only disabled accounts.
'''
LOCAL = 'local accounts'
EXT = 'external accounts'
data = dict()
data[LOCAL] = self._get_local_users(disabled=kwargs.get('disabled'))
data[EXT] = self._get_external_accounts(data[LOCAL].keys()) or 'N/A'
data['local groups'] = self._get_local_groups()
return data
def _system(self, *args, **kwargs):
'''
This basically calls grains items and picks out only
necessary information in a certain structure.
:param args:
:param kwargs:
:return:
'''
sysinfo = SysInfo(__grains__.get("kernel"))
data = dict()
data['cpu'] = sysinfo._get_cpu()
data['disks'] = sysinfo._get_fs()
data['mounts'] = sysinfo._get_mounts()
data['memory'] = sysinfo._get_mem()
data['network'] = sysinfo._get_network()
data['os'] = sysinfo._get_os()
return data
def _software(self, *args, **kwargs):
'''
Return installed software.
'''
data = dict()
if 'exclude' in kwargs:
excludes = kwargs['exclude'].split(",")
else:
excludes = list()
os_family = __grains__.get("os_family").lower()
# Get locks
if os_family == 'suse':
LOCKS = "pkg.list_locks"
if 'products' not in excludes:
products = __salt__['pkg.list_products']()
if products:
data['products'] = products
elif os_family == 'redhat':
LOCKS = "pkg.get_locked_packages"
else:
LOCKS = None
if LOCKS and 'locks' not in excludes:
locks = __salt__[LOCKS]()
if locks:
data['locks'] = locks
# Get patterns
if os_family == 'suse':
PATTERNS = 'pkg.list_installed_patterns'
elif os_family == 'redhat':
PATTERNS = 'pkg.group_list'
else:
PATTERNS = None
if PATTERNS and 'patterns' not in excludes:
patterns = __salt__[PATTERNS]()
if patterns:
data['patterns'] = patterns
# Get packages
if 'packages' not in excludes:
data['packages'] = __salt__['pkg.list_pkgs']()
# Get repositories
if 'repositories' not in excludes:
repos = __salt__['pkg.list_repos']()
if repos:
data['repositories'] = repos
return data
def _services(self, *args, **kwargs):
'''
Get list of enabled and disabled services on the particular system.
'''
return {
'enabled': __salt__['service.get_enabled'](),
'disabled': __salt__['service.get_disabled'](),
}
def _id_resolv(self, iid, named=True, uid=True):
'''
Resolve local users and groups.
:param iid:
:param named:
:param uid:
:return:
'''
if not self.local_identity:
self.local_identity['users'] = self._get_local_users()
self.local_identity['groups'] = self._get_local_groups()
if not named:
return iid
for name, meta in self.local_identity[uid and 'users'
or 'groups'].items():
if (uid and int(meta.get('uid', -1)) == iid) or (not uid and int(
meta.get('gid', -1)) == iid):
return name
return iid
def _payload(self, *args, **kwargs):
'''
Find all unmanaged files.
Parameters:
* **filter**: Include only results which path starts from the filter string.
* **time**: Display time in Unix ticks or format according to the configured TZ (default)
Values: ticks, tz (default)
* **size**: Format size. Values: B, KB, MB, GB
* **owners**: Resolve UID/GID to an actual names or leave them numeric (default).
Values: name (default), id
* **type**: Comma-separated type of included payload: dir (or directory), link and/or file.
* **brief**: Return just a list of matches, if True. Default: False
'''
def _size_format(size, fmt):
if fmt is None:
return size
fmt = fmt.lower()
if fmt == "b":
return "{0} Bytes".format(size)
elif fmt == "kb":
return "{0} Kb".format(round((float(size) / 0x400), 2))
elif fmt == "mb":
return "{0} Mb".format(round((float(size) / 0x400 / 0x400), 2))
elif fmt == "gb":
return "{0} Gb".format(
round((float(size) / 0x400 / 0x400 / 0x400), 2))
filter = None
if 'filter' in kwargs:
filter = kwargs['filter']
timeformat = kwargs.get("time", "tz")
if timeformat not in ["ticks", "tz"]:
raise InspectorQueryException(
'Unknown "{0}" value for parameter "time"'.format(timeformat))
tfmt = lambda param: timeformat == "tz" and time.strftime(
"%b %d %Y %H:%M:%S", time.gmtime(param)) or int(param)
size_fmt = kwargs.get("size")
if size_fmt is not None and size_fmt.lower() not in [
"b", "kb", "mb", "gb"
]:
raise InspectorQueryException(
'Unknown "{0}" value for parameter "size". '
'Should be either B, Kb, Mb or Gb'.format(timeformat))
owners = kwargs.get("owners", "id")
if owners not in ["name", "id"]:
raise InspectorQueryException(
'Unknown "{0}" value for parameter "owners". '
'Should be either name or id (default)'.format(owners))
incl_type = [
prm for prm in kwargs.get("type", "").lower().split(",") if prm
]
if not incl_type:
incl_type.append("file")
for i_type in incl_type:
if i_type not in [
"directory", "dir", "d", "file", "f", "link", "l"
]:
raise InspectorQueryException(
'Unknown "{0}" values for parameter "type". '
'Should be comma separated one or more of '
'dir, file and/or link.'.format(", ".join(incl_type)))
where_clause = set()
for i_type in incl_type:
if i_type in ["file", "f"]:
where_clause.add("p_type = 'f'")
elif i_type in ["d", "dir", "directory"]:
where_clause.add("p_type = 'd'")
elif i_type in ["l", "link"]:
where_clause.add("p_type = 'l'")
if filter:
where_filter_clause = " AND path LIKE '{0}%'".format(filter)
else:
where_filter_clause = ""
self.db.open()
self.db.cursor.execute(
"SELECT id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime "
"FROM inspector_payload "
"WHERE {0}{1}".format(" OR ".join(list(where_clause)),
where_filter_clause))
brief = kwargs.get("brief")
if brief:
data = list()
else:
data = dict()
for pld_data in self.db.cursor.fetchall():
p_id, path, p_type, mode, uid, gid, p_size, atime, mtime, ctime = pld_data
if brief:
data.append(path)
else:
data[path] = {
'uid': self._id_resolv(uid, named=(owners == "id")),
'gid': self._id_resolv(gid,
named=(owners == "id"),
uid=False),
'size': _size_format(p_size, fmt=size_fmt),
'mode': oct(mode),
'accessed': tfmt(atime),
'modified': tfmt(mtime),
'created': tfmt(ctime),
}
self.db.close()
return data
def _all(self, *args, **kwargs):
'''
Return all the summary of the particular system.
'''
data = dict()
data['software'] = self._software(**kwargs)
data['system'] = self._system(**kwargs)
data['services'] = self._services(**kwargs)
try:
data['configuration'] = self._configuration(**kwargs)
except InspectorQueryException as ex:
data['configuration'] = 'N/A'
log.error(ex)
data['payload'] = self._payload(**kwargs) or 'N/A'
return data