def _setup_cached_uuid():
""" Get the cached uuid and cached system uui path, read the files
and remove the cached uuid """
# Check for a cloned system with existing hubble_uuid
def _get_uuid_from_system():
query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv'
# Prefer our /opt/osquery/osqueryi if present
osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi',
'/usr/bin/osqueryi')
for path in osqueryipaths:
if hubblestack.utils.path.which(path):
live_uuid = hubblestack.modules.cmdmod.run_stdout(
'{0} {1}'.format(path, query), output_loglevel='quiet')
live_uuid = str(live_uuid).upper()
if len(live_uuid) == 36:
return live_uuid
return None
# If osquery isn't available, attempt to get uuid from /sys path (linux only)
try:
with open('/sys/devices/virtual/dmi/id/product_uuid',
'r') as product_uuid_file:
file_uuid = product_uuid_file.read()
file_uuid = str(file_uuid).upper()
if len(file_uuid) == 36:
return file_uuid
return None
except Exception:
return None
cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']),
'hubble_cached_uuid')
cached_system_uuid_path = os.path.join(
os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid')
try:
if os.path.isfile(cached_uuid_path) and os.path.isfile(
cached_system_uuid_path):
with open(cached_uuid_path, 'r') as cached_uuid_file, \
open(cached_system_uuid_path, 'r') as cached_system_uuid_file:
cached_uuid = cached_uuid_file.read()
cached_system_uuid = cached_system_uuid_file.read()
if cached_uuid != cached_system_uuid:
live_uuid = _get_uuid_from_system()
if live_uuid != cached_system_uuid:
log.error(
"potentially cloned system detected: System_uuid grain "
"previously saved on disk doesn't match live system value.\n"
"Resettig cached hubble_uuid value.")
os.remove(cached_uuid_path)
except Exception:
log.exception(
"Problem opening cache files while checking for previously cloned system"
)
def run_function():
'''
Run a single function requested by the user
'''
# Parse the args
args = []
kwargs = {}
for arg in __opts__['args']:
if '=' in arg:
kwarg, _, value = arg.partition('=')
kwargs[kwarg] = value
else:
args.append(arg)
log.debug('Parsed args: {0} | Parsed kwargs: {1}'.format(args, kwargs))
log.info('Executing user-requested function {0}'.format(
__opts__['function']))
try:
ret = __salt__[__opts__['function']](*args, **kwargs)
except KeyError:
log.error('Function {0} is not available, or not valid.'.format(
__opts__['function']))
sys.exit(1)
if __opts__['return']:
returner = '{0}.returner'.format(__opts__['return'])
if returner not in __returners__:
log.error('Could not find {0} returner.'.format(returner))
else:
log.info('Returning job data to {0}'.format(returner))
returner_ret = {
'id': __grains__['id'],
'jid': salt.utils.jid.gen_jid(__opts__),
'fun': __opts__['function'],
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret,
'retry': False
}
if __opts__.get('returner_retry', False):
returner_ret['retry'] = True
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__['json_print']:
print(json.dumps(ret))
else:
if not __opts__['no_pprint']:
pprint.pprint(ret)
else:
print(ret)
def run_function():
"""
Run a single function requested by the user
"""
# Parse the args
args = []
kwargs = {}
for arg in __opts__['args']:
if '=' in arg:
kwarg, _, value = arg.partition('=')
kwargs[kwarg] = value
else:
args.append(arg)
log.debug('Parsed args: %s | Parsed kwargs: %s', args, kwargs)
log.info('Executing user-requested function %s', __opts__['function'])
mod_fun = __mods__.get(__opts__['function'])
if not mod_fun or not callable(mod_fun):
log.error('Function %s is not available, or not valid.',
__opts__['function'])
sys.exit(1)
ret = mod_fun(*args, **kwargs)
if __opts__['return']:
returner = '{0}.returner'.format(__opts__['return'])
if returner not in __returners__:
log.error('Could not find %s returner.', returner)
else:
log.info('Returning job data to %s', returner)
returner_ret = {
'id': __grains__['id'],
'jid': hubblestack.utils.jid.gen_jid(__opts__),
'fun': __opts__['function'],
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret
}
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__['json_print']:
print(json.dumps(ret))
else:
if not __opts__['no_pprint']:
pprint.pprint(ret)
else:
print(ret)
def run_function():
"""
Run a single function requested by the user
"""
# Parse the args
args = []
kwargs = {}
for arg in __opts__["args"]:
if "=" in arg:
kwarg, _, value = arg.partition("=")
kwargs[kwarg] = value
else:
args.append(arg)
log.debug("Parsed args: %s | Parsed kwargs: %s", args, kwargs)
log.info("Executing user-requested function %s", __opts__["function"])
mod_fun = __mods__.get(__opts__["function"])
if not mod_fun or not callable(mod_fun):
log.error("Function %s is not available, or not valid.",
__opts__["function"])
sys.exit(1)
ret = mod_fun(*args, **kwargs)
if __opts__["return"]:
returner = "{0}.returner".format(__opts__["return"])
if returner not in __returners__:
log.error("Could not find %s returner.", returner)
else:
log.info("Returning job data to %s", returner)
returner_ret = {
"id": __grains__["id"],
"jid": hubblestack.utils.jid.gen_jid(__opts__),
"fun": __opts__["function"],
"fun_args": args + ([kwargs] if kwargs else []),
"return": ret,
}
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__["json_print"]:
print(json.dumps(ret))
else:
if not __opts__["no_pprint"]:
pprint.pprint(ret)
else:
print(ret)
def publish(report_directly_to_splunk=True, remove_dots=True, *args):
"""
Publishes config to splunk at an interval defined in schedule
report_directly_to_splunk
Whether to emit directly to splunk in addition to returning as a normal
job. Defaults to True.
remove_dots
Whether to replace dots in top-level keys with underscores for ease
of handling in splunk. Defaults to True.
*args
Tuple of opts to log (keys in __opts__). Only those key-value pairs
would be published, keys for which are in *args If not passed, entire
__opts__ (excluding password/token) would be published
"""
log.debug("Started publishing config to splunk")
opts_to_log = {}
if not args:
opts_to_log = copy.deepcopy(__opts__)
if "grains" in opts_to_log:
opts_to_log.pop("grains")
else:
for arg in args:
if arg in __opts__:
opts_to_log[arg] = __opts__[arg]
# 'POP' is for tracking persistent opts protection
if os.environ.get("NOISY_POP_DEBUG"):
log.error("POP config_publish (id=%d)", id(__opts__))
filtered_conf = hubblestack.log.filter_logs(opts_to_log,
remove_dots=remove_dots)
if report_directly_to_splunk:
hubblestack.log.emit_to_splunk(filtered_conf, "INFO",
"hubblestack.hubble_config")
log.debug("Published config to splunk")
return filtered_conf
def _execute_function(jobdata, func, returners, args, kwargs):
""" Run the scheduled function """
log.debug('Executing scheduled function %s', func)
jobdata['last_run'] = time.time()
ret = __salt__[func](*args, **kwargs)
if __opts__['log_level'] == 'debug':
log.debug('Job returned:\n%s', ret)
for returner in returners:
returner = '{0}.returner'.format(returner)
if returner not in __returners__:
log.error('Could not find %s returner.', returner)
continue
log.debug('Returning job data to %s', returner)
returner_ret = {'id': __grains__['id'],
'jid': salt.utils.jid.gen_jid(__opts__),
'fun': func,
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret}
__returners__[returner](returner_ret)
def _execute_function(jobdata, func, returners, args, kwargs):
"""Run the scheduled function"""
log.debug("Executing scheduled function %s", func)
jobdata["last_run"] = time.time()
ret = __mods__[func](*args, **kwargs)
if __opts__["log_level"] == "debug":
log.debug("Job returned:\n%s", ret)
for returner in returners:
returner = "{0}.returner".format(returner)
if returner not in __returners__:
log.error("Could not find %s returner.", returner)
continue
log.debug("Returning job data to %s", returner)
returner_ret = {
"id": __grains__["id"],
"jid": hubblestack.utils.jid.gen_jid(__opts__),
"fun": func,
"fun_args": args + ([kwargs] if kwargs else []),
"return": ret,
}
__returners__[returner](returner_ret)
def schedule():
"""
Rudimentary single-pass scheduler
If we find we miss some of the salt scheduler features we could potentially
pull in some of that code.
Schedule data should be placed in the config with the following format:
.. code-block:: yaml
schedule:
job1:
function: hubble.audit
seconds: 3600
splay: 100
min_splay: 50
args:
- cis.centos-7-level-1-scored-v2-1-0
kwargs:
verbose: True
show_profile: True
returner: splunk_nova_return
run_on_start: True
Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a
scheduled job must always have a ``function`` and a time in ``seconds`` of
how often to run the job.
function
Function to run in the format ``<module>.<function>``. Technically any
salt module can be run in this way, but we recommend sticking to hubble
functions. For simplicity, functions are run in the main daemon thread,
so overloading the scheduler can result in functions not being run in
a timely manner.
seconds
Frequency with which the job should be run, in seconds
splay
Randomized splay for the job, in seconds. A random number between <min_splay> and
<splay> will be chosen and added to the ``seconds`` argument, to decide
the true frequency. The splay will be chosen on first run, and will
only change when the daemon is restarted. Optional.
min_splay
This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random
between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it
defaults to zero. Optional.
args
List of arguments for the function. Optional.
kwargs
Dict of keyword arguments for the function. Optional.
returner
String with a single returner, or list of returners to which we should
send the results. Optional.
run_on_start
Whether to run the scheduled job on daemon start. Defaults to False. Optional.
"""
sf_count = 0
base = datetime(2018, 1, 1, 0, 0)
schedule_config = __opts__.get('schedule', {})
if 'user_schedule' in __opts__ and isinstance(__opts__['user_schedule'],
dict):
schedule_config.update(__opts__['user_schedule'])
for jobname, jobdata in schedule_config.items():
try:
# Error handling galore
if not jobdata or not isinstance(jobdata, dict):
log.error('Scheduled job %s does not have valid data', jobname)
continue
if 'function' not in jobdata or 'seconds' not in jobdata:
log.error(
'Scheduled job %s is missing a ``function`` or ``seconds`` argument',
jobname)
continue
func = jobdata['function']
if func not in __mods__:
log.error(
'Scheduled job %s has a function %s which could not be found.',
jobname, func)
continue
try:
if 'cron' in jobdata:
seconds = getsecondsbycronexpression(base, jobdata['cron'])
else:
seconds = int(jobdata['seconds'])
splay = int(jobdata.get('splay', 0))
min_splay = int(jobdata.get('min_splay', 0))
except ValueError:
log.error(
'Scheduled job %s has an invalid value for seconds or splay.',
jobname)
args = jobdata.get('args', [])
if not isinstance(args, list):
log.error('Scheduled job %s has args not formed as a list: %s',
jobname, args)
kwargs = jobdata.get('kwargs', {})
if not isinstance(kwargs, dict):
log.error(
'Scheduled job %s has kwargs not formed as a dict: %s',
jobname, kwargs)
returners = jobdata.get('returner', [])
if not isinstance(returners, list):
returners = [returners]
# Actually process the job
run = _process_job(jobdata, splay, seconds, min_splay, base)
if run:
_execute_function(jobdata, func, returners, args, kwargs)
sf_count += 1
except:
log.error(
"Exception in running job: %s; continuing with next job...",
jobname,
exc_info=True)
return sf_count
def kill_other_or_sys_exit(xpid,
hname=r'hubble',
ksig=signal.SIGTERM,
kill_other=True,
no_pgrp=True):
""" Attempt to locate other hubbles using a cmdline regular expression and kill them when found.
If killing the other processes fails (or kill_other is False), sys.exit instead.
params:
hname :- the regular expression pattern to use to locate hubble (default: hubble)
ksig :- the signal to use to kill the other processes (default: signal.SIGTERM=15)
kill_other :- (default: True); when false, don't attempt to kill,
just locate and exit (if found)
no_pgrp :- Avoid killing processes in this pgrp (avoid suicide). When no_pgrp is True,
invoke os.getprgp() to populate the actual value.
caveats:
There are some detailed notes on the process scanning in the function as comments.
The most important caveat is that the hname regular expressions must match expecting
that /proc/$$/cmdline text is null separated, not space separated.
The other main caveat is that we can't actually examine the /proc/$$/exe file (that's
always just a python). We have to scan the invocation text the kernel stored at launch.
That text is not immutable and should not (normally) be relied upon for any purpose
-- and this method does rely on it.
"""
if no_pgrp is True:
no_pgrp = os.getpgrp()
if isinstance(no_pgrp, int):
no_pgrp = str(no_pgrp)
if os.path.isdir("/proc/{pid}".format(pid=xpid)):
# NOTE: we'd prefer to check readlink(/proc/[pid]/exe), but that won't do
# any good the /opt/whatever/bin/hubble is normally a text file with a
# shebang; which the kernel picks up and uses to execute the real binary
# with the "bin" file as an argument; so we'll have to live with cmdline
pfile = '/proc/{pid}/cmdline'.format(pid=xpid)
log.debug('searching %s for hubble procs matching %s', pfile, hname)
with open(pfile, 'r') as pidfile:
# NOTE: cmdline is actually null separated, not space separated
# that shouldn't matter much for most hname regular expressions, but one never knows.
cmdline = pidfile.readline().replace('\x00', ' ').strip()
if re.search(hname, cmdline):
if no_pgrp:
pstatfile = '/proc/{pid}/stat'.format(pid=xpid)
with open(pstatfile, 'r') as fh2:
# NOTE: man proc(5) § /proc/[pid]/stat
# (pid, comm, state, ppid, pgrp, session, tty_nr, tpgid, flags, ...)
pgrp = fh2.readline().split()[4]
if pgrp == no_pgrp:
log.debug(
"process (%s) exists and seems to be a hubble, "
"but matches our process group (%s), ignoring",
xpid, pgrp)
return False
if kill_other:
log.warn(
"process seems to still be alive and seems to be hubble,"
" attempting to shutdown")
os.kill(int(xpid), ksig)
time.sleep(1)
if os.path.isdir("/proc/{pid}".format(pid=xpid)):
log.error(
"fatal error: failed to shutdown process (pid=%s) successfully",
xpid)
sys.exit(1)
else:
return True
else:
log.error(
"refusing to run while another hubble instance is running")
sys.exit(1)
else:
# pidfile present, but nothing at that pid. Did we receive a sigterm?
log.warning(
'Pidfile found on startup, but no process at that pid. Did hubble receive a SIGTERM?'
)
return False
def load_config():
'''
Load the config from configfile and load into imported salt modules
'''
# Parse arguments
parsed_args = parse_args()
# Let's find out the path of this module
if 'SETUP_DIRNAME' in globals():
# This is from the exec() call in Salt's setup.py
this_file = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py') # pylint: disable=E0602
else:
this_file = __file__
install_dir = os.path.dirname(os.path.realpath(this_file))
# Load unique data for Windows or Linux
if salt.utils.platform.is_windows():
if parsed_args.get('configfile') is None:
parsed_args[
'configfile'] = 'C:\\Program Files (x86)\\Hubble\\etc\\hubble\\hubble.conf'
salt.config.DEFAULT_MINION_OPTS[
'cachedir'] = 'C:\\Program Files (x86)\\hubble\\var\\cache'
salt.config.DEFAULT_MINION_OPTS[
'pidfile'] = 'C:\\Program Files (x86)\\hubble\\var\\run\\hubble.pid'
salt.config.DEFAULT_MINION_OPTS[
'log_file'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble.log'
salt.config.DEFAULT_MINION_OPTS[
'osquery_dbpath'] = 'C:\\Program Files (x86)\\hubble\\var\\hubble_osquery_db'
salt.config.DEFAULT_MINION_OPTS[
'osquerylogpath'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery'
salt.config.DEFAULT_MINION_OPTS['osquerylog_backupdir'] = \
'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery\\backuplogs'
else:
if parsed_args.get('configfile') is None:
parsed_args['configfile'] = '/etc/hubble/hubble'
salt.config.DEFAULT_MINION_OPTS['cachedir'] = '/var/cache/hubble'
salt.config.DEFAULT_MINION_OPTS['pidfile'] = '/var/run/hubble.pid'
salt.config.DEFAULT_MINION_OPTS['log_file'] = '/var/log/hubble'
salt.config.DEFAULT_MINION_OPTS[
'osquery_dbpath'] = '/var/cache/hubble/osquery'
salt.config.DEFAULT_MINION_OPTS[
'osquerylogpath'] = '/var/log/hubble_osquery'
salt.config.DEFAULT_MINION_OPTS[
'osquerylog_backupdir'] = '/var/log/hubble_osquery/backuplogs'
salt.config.DEFAULT_MINION_OPTS['file_roots'] = {'base': []}
salt.config.DEFAULT_MINION_OPTS['log_level'] = 'error'
salt.config.DEFAULT_MINION_OPTS['file_client'] = 'local'
salt.config.DEFAULT_MINION_OPTS[
'fileserver_update_frequency'] = 43200 # 12 hours
salt.config.DEFAULT_MINION_OPTS[
'grains_refresh_frequency'] = 3600 # 1 hour
salt.config.DEFAULT_MINION_OPTS['scheduler_sleep_frequency'] = 0.5
salt.config.DEFAULT_MINION_OPTS['default_include'] = 'hubble.d/*.conf'
salt.config.DEFAULT_MINION_OPTS['logfile_maxbytes'] = 100000000 # 100MB
salt.config.DEFAULT_MINION_OPTS[
'logfile_backups'] = 1 # maximum rotated logs
salt.config.DEFAULT_MINION_OPTS[
'delete_inaccessible_azure_containers'] = False
salt.config.DEFAULT_MINION_OPTS[
'enable_globbing_in_nebula_masking'] = False # Globbing will not be supported in nebula masking
salt.config.DEFAULT_MINION_OPTS[
'osquery_logfile_maxbytes'] = 50000000 # 50MB
salt.config.DEFAULT_MINION_OPTS[
'osquery_logfile_maxbytes_toparse'] = 100000000 #100MB
salt.config.DEFAULT_MINION_OPTS['osquery_backuplogs_count'] = 2
global __opts__
__opts__ = salt.config.minion_config(parsed_args.get('configfile'))
__opts__.update(parsed_args)
__opts__['conf_file'] = parsed_args.get('configfile')
__opts__['install_dir'] = install_dir
if __opts__['version']:
print(__version__)
clean_up_process(None, None)
sys.exit(0)
if __opts__['buildinfo']:
try:
from hubblestack import __buildinfo__
except ImportError:
__buildinfo__ = 'NOT SET'
print(__buildinfo__)
clean_up_process(None, None)
sys.exit(0)
scan_proc = __opts__.get('scan_proc', False)
if __opts__['daemonize']:
# before becoming a daemon, check for other procs and possibly send
# them a signal 15 (otherwise refuse to run)
if not __opts__.get('ignore_running', False):
check_pidfile(kill_other=True, scan_proc=scan_proc)
salt.utils.daemonize()
create_pidfile()
elif not __opts__['function'] and not __opts__['version'] and not __opts__[
'buildinfo']:
# check the pidfile and possibly refuse to run
# (assuming this isn't a single function call)
if not __opts__.get('ignore_running', False):
check_pidfile(kill_other=False, scan_proc=scan_proc)
signal.signal(signal.SIGTERM, clean_up_process)
signal.signal(signal.SIGINT, clean_up_process)
signal.signal(signal.SIGABRT, clean_up_process)
signal.signal(signal.SIGFPE, clean_up_process)
signal.signal(signal.SIGILL, clean_up_process)
signal.signal(signal.SIGSEGV, clean_up_process)
if not salt.utils.platform.is_windows():
signal.signal(signal.SIGHUP, clean_up_process)
signal.signal(signal.SIGQUIT, clean_up_process)
# Optional sleep to wait for network
time.sleep(int(__opts__.get('startup_sleep', 0)))
# Convert -vvv to log level
if __opts__['log_level'] is None:
# Default to 'error'
__opts__['log_level'] = 'error'
# Default to more verbose if we're daemonizing
if __opts__['daemonize']:
__opts__['log_level'] = 'info'
# Handle the explicit -vvv settings
if __opts__['verbose'] == 1:
__opts__['log_level'] = 'warning'
elif __opts__['verbose'] == 2:
__opts__['log_level'] = 'info'
elif __opts__['verbose'] >= 3:
__opts__['log_level'] = 'debug'
# Setup module/grain/returner dirs
module_dirs = __opts__.get('module_dirs', [])
module_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'modules'))
__opts__['module_dirs'] = module_dirs
grains_dirs = __opts__.get('grains_dirs', [])
grains_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'grains'))
__opts__['grains_dirs'] = grains_dirs
returner_dirs = __opts__.get('returner_dirs', [])
returner_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'returners'))
__opts__['returner_dirs'] = returner_dirs
fileserver_dirs = __opts__.get('fileserver_dirs', [])
fileserver_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'fileserver'))
__opts__['fileserver_dirs'] = fileserver_dirs
utils_dirs = __opts__.get('utils_dirs', [])
utils_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'utils'))
__opts__['utils_dirs'] = utils_dirs
fdg_dirs = __opts__.get('fdg_dirs', [])
fdg_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'fdg'))
__opts__['fdg_dirs'] = fdg_dirs
__opts__['file_roots']['base'].insert(
0, os.path.join(os.path.dirname(__file__), 'files'))
if 'roots' not in __opts__['fileserver_backend']:
__opts__['fileserver_backend'].append('roots')
# Disable all of salt's boto modules, they give nothing but trouble to the loader
disable_modules = __opts__.get('disable_modules', [])
disable_modules.extend([
'boto3_elasticache',
'boto3_route53',
'boto3_sns',
'boto_apigateway',
'boto_asg',
'boto_cfn',
'boto_cloudfront',
'boto_cloudtrail',
'boto_cloudwatch_event',
'boto_cloudwatch',
'boto_cognitoidentity',
'boto_datapipeline',
'boto_dynamodb',
'boto_ec2',
'boto_efs',
'boto_elasticache',
'boto_elasticsearch_domain',
'boto_elb',
'boto_elbv2',
'boto_iam',
'boto_iot',
'boto_kinesis',
'boto_kms',
'boto_lambda',
'boto_rds',
'boto_route53',
'boto_s3_bucket',
'boto_s3',
'boto_secgroup',
'boto_sns',
'boto_sqs',
'boto_ssm',
'boto_vpc',
])
__opts__['disable_modules'] = disable_modules
# Console logging is probably the same, but can be different
console_logging_opts = {
'log_level':
__opts__.get('console_log_level', __opts__['log_level']),
'log_format':
__opts__.get('console_log_format',
'%(asctime)s [%(levelname)-5s] %(message)s'),
'date_format':
__opts__.get('console_log_date_format', '%H:%M:%S'),
}
file_logging_opts = {
'log_file':
__opts__.get('log_file', '/var/log/hubble'),
'log_level':
__opts__['log_level'],
'log_format':
__opts__.get(
'log_format',
'%(asctime)s,%(msecs)03d [%(levelname)-5s] [%(name)s:%(lineno)d] %(message)s'
),
'date_format':
__opts__.get('log_date_format', '%Y-%m-%d %H:%M:%S'),
'max_bytes':
__opts__.get('logfile_maxbytes', 100000000),
'backup_count':
__opts__.get('logfile_backups', 1),
}
# Setup logging
hubblestack.log.setup_console_logger(**console_logging_opts)
hubblestack.log.setup_file_logger(**file_logging_opts)
# 384 is 0o600 permissions, written without octal for python 2/3 compat
os.chmod(__opts__['log_file'], 384)
os.chmod(parsed_args.get('configfile'), 384)
# Check for a cloned system with existing hubble_uuid
def _get_uuid_from_system():
query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv'
# Prefer our /opt/osquery/osqueryi if present
osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi',
'/usr/bin/osqueryi')
for path in osqueryipaths:
if salt.utils.path.which(path):
live_uuid = salt.modules.cmdmod.run_stdout(
'{0} {1}'.format(path, query), output_loglevel='quiet')
live_uuid = str(live_uuid).upper()
if len(live_uuid) == 36:
return live_uuid
else:
return None
# If osquery isn't available, attempt to get uuid from /sys path (linux only)
try:
with open('/sys/devices/virtual/dmi/id/product_uuid', 'r') as f:
file_uuid = f.read()
file_uuid = str(file_uuid).upper()
if len(file_uuid) == 36:
return file_uuid
else:
return None
except Exception:
return None
cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']),
'hubble_cached_uuid')
cached_system_uuid_path = os.path.join(
os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid')
try:
if os.path.isfile(cached_uuid_path) and os.path.isfile(
cached_system_uuid_path):
with open(cached_uuid_path,
'r') as f, open(cached_system_uuid_path, 'r') as g:
cached_uuid = f.read()
cached_system_uuid = g.read()
if cached_uuid != cached_system_uuid:
live_uuid = _get_uuid_from_system()
if live_uuid != cached_system_uuid:
log.error(
"potentially cloned system detected: System_uuid grain "
"previously saved on disk doesn't match live system value.\n"
"Resettig cached hubble_uuid value.")
os.remove(cached_uuid_path)
except Exception:
log.exception(
"Problem opening cache files while checking for previously cloned system"
)
refresh_grains(initial=True)
if __salt__['config.get']('splunklogging', False):
hubblestack.log.setup_splunk_logger()
hubblestack.log.emit_to_splunk(__grains__,
'INFO',
'hubblestack.grains_report',
remove_sensitive_logs=True)
def schedule():
'''
Rudimentary single-pass scheduler
If we find we miss some of the salt scheduler features we could potentially
pull in some of that code.
Schedule data should be placed in the config with the following format:
.. code-block:: yaml
schedule:
job1:
function: hubble.audit
seconds: 3600
splay: 100
min_splay: 50
args:
- cis.centos-7-level-1-scored-v2-1-0
kwargs:
verbose: True
show_profile: True
returner: splunk_nova_return
run_on_start: True
Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a
scheduled job must always have a ``function`` and a time in ``seconds`` of
how often to run the job.
function
Function to run in the format ``<module>.<function>``. Technically any
salt module can be run in this way, but we recommend sticking to hubble
functions. For simplicity, functions are run in the main daemon thread,
so overloading the scheduler can result in functions not being run in
a timely manner.
seconds
Frequency with which the job should be run, in seconds
splay
Randomized splay for the job, in seconds. A random number between <min_splay> and
<splay> will be chosen and added to the ``seconds`` argument, to decide
the true frequency. The splay will be chosen on first run, and will
only change when the daemon is restarted. Optional.
min_splay
This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random
between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it
defaults to zero. Optional.
args
List of arguments for the function. Optional.
kwargs
Dict of keyword arguments for the function. Optional.
returner
String with a single returner, or list of returners to which we should
send the results. Optional.
run_on_start
Whether to run the scheduled job on daemon start. Defaults to False.
Optional.
'''
sf_count = 0
base = datetime(2018, 1, 1, 0, 0)
schedule_config = __opts__.get('schedule', {})
if 'user_schedule' in __opts__ and isinstance(__opts__['user_schedule'],
dict):
schedule_config.update(__opts__['user_schedule'])
for jobname, jobdata in schedule_config.iteritems():
# Error handling galore
if not jobdata or not isinstance(jobdata, dict):
log.error(
'Scheduled job {0} does not have valid data'.format(jobname))
continue
if 'function' not in jobdata or 'seconds' not in jobdata:
log.error('Scheduled job {0} is missing a ``function`` or '
'``seconds`` argument'.format(jobname))
continue
func = jobdata['function']
if func not in __salt__:
log.error('Scheduled job {0} has a function {1} which could not '
'be found.'.format(jobname, func))
continue
try:
if 'cron' in jobdata:
seconds = getsecondsbycronexpression(base, jobdata['cron'])
else:
seconds = int(jobdata['seconds'])
splay = int(jobdata.get('splay', 0))
min_splay = int(jobdata.get('min_splay', 0))
except ValueError:
log.error('Scheduled job {0} has an invalid value for seconds or '
'splay.'.format(jobname))
args = jobdata.get('args', [])
if not isinstance(args, list):
log.error(
'Scheduled job {0} has args not formed as a list: {1}'.format(
jobname, args))
kwargs = jobdata.get('kwargs', {})
if not isinstance(kwargs, dict):
log.error('Scheduled job {0} has kwargs not formed as a dict: {1}'.
format(jobname, kwargs))
returners = jobdata.get('returner', [])
if not isinstance(returners, list):
returners = [returners]
returner_retry = jobdata.get('returner_retry', False)
# Actually process the job
run = False
if 'last_run' not in jobdata:
if jobdata.get('run_on_start', False):
if splay:
# Run `splay` seconds in the future, by telling the scheduler we last ran it
# `seconds - splay` seconds ago.
jobdata['last_run'] = time.time() - (
seconds - random.randint(min_splay, splay))
else:
# Run now
run = True
jobdata['last_run'] = time.time()
else:
if splay:
# Run `seconds + splay` seconds in the future by telling the scheduler we last
# ran it at now + `splay` seconds.
jobdata['last_run'] = time.time() + random.randint(
min_splay, splay)
elif 'buckets' in jobdata:
# Place the host in a bucket and fix the execution time.
jobdata['last_run'] = getlastrunbybuckets(
jobdata['buckets'], seconds)
log.debug('last_run according to bucket is {0}'.format(
jobdata['last_run']))
elif 'cron' in jobdata:
# execute the hubble process based on cron expression
jobdata['last_run'] = getlastrunbycron(base, seconds)
else:
# Run in `seconds` seconds.
jobdata['last_run'] = time.time()
if jobdata['last_run'] < time.time() - seconds:
run = True
if run:
log.debug('Executing scheduled function {0}'.format(func))
jobdata['last_run'] = time.time()
ret = __salt__[func](*args, **kwargs)
sf_count += 1
if __opts__['log_level'] == 'debug':
log.debug('Job returned:\n{0}'.format(ret))
for returner in returners:
returner = '{0}.returner'.format(returner)
if returner not in __returners__:
log.error('Could not find {0} returner.'.format(returner))
continue
log.debug('Returning job data to {0}'.format(returner))
returner_ret = {
'id': __grains__['id'],
'jid': salt.utils.jid.gen_jid(__opts__),
'fun': func,
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret,
'retry': returner_retry
}
__returners__[returner](returner_ret)
return sf_count
def refresh_grains(initial=False):
"""
Refresh the grains, pillar, utils, modules, and returners
"""
global __opts__
global __grains__
global __utils__
global __mods__
global __pillar__
global __returners__
global __context__
# 'POP' is for tracking persistent opts protection
if os.environ.get("NOISY_POP_DEBUG"):
log.error("POP refreshing grains (id=%d)", id(__opts__))
persist, old_grains = {}, {}
if initial:
if not os.environ.get("NO_PRESERVE_OPTS"):
if os.environ.get("NOISY_POP_DEBUG"):
log.error("POP setting __opts__ to preservable (id=%d)",
id(__opts__))
hubblestack.loader.set_preservable_opts(__opts__)
elif os.environ.get("NOISY_POP_DEBUG"):
log.error(
"POP we are not attemting to protect __opts__ from lazyloader reloads"
)
else:
old_grains = copy.deepcopy(__grains__)
for grain in __opts__.get("grains_persist", []):
if grain in __grains__:
persist[grain] = __grains__[grain]
# Hardcode these core grains as persisting
persist = {
grain: __grains__[grain]
for grain in ["hubble_version", "buildinfo"] if grain in __grains__
}
if initial:
__context__ = {}
if "grains" in __opts__:
__opts__.pop("grains")
if "pillar" in __opts__:
__opts__.pop("pillar")
__grains__ = hubblestack.loader.grains(__opts__)
__grains__.update(persist)
__grains__["session_uuid"] = SESSION_UUID
# This was a weird one. In older versions of hubble the version and
# buildinfo were not persisted automatically which means that if you
# installed a new version without restarting hubble, grains refresh could
# cause that old daemon to report grains as if it were the new version.
# Now if this hubble_marker_3 grain is present you know you can trust the
# hubble_version and buildinfo.
__grains__["hubble_marker_3"] = True
old_grains.update(__grains__)
__grains__ = old_grains
# Check for default gateway and fall back if necessary
if __grains__.get(
"ip_gw",
None) is False and "fallback_fileserver_backend" in __opts__:
log.info(
"No default gateway detected; using fallback_fileserver_backend.")
__opts__["fileserver_backend"] = __opts__[
"fallback_fileserver_backend"]
__opts__["hubble_uuid"] = __grains__.get("hubble_uuid", None)
__opts__["system_uuid"] = __grains__.get("system_uuid", None)
__pillar__ = {}
__opts__["grains"] = __grains__
__opts__["pillar"] = __pillar__
__utils__ = hubblestack.loader.utils(__opts__)
__mods__ = hubblestack.loader.modules(__opts__,
utils=__utils__,
context=__context__)
__returners__ = hubblestack.loader.returners(__opts__, __mods__)
# the only things that turn up in here (and that get preserved)
# are pulsar.queue, pulsar.notifier and cp.fileclient_###########
# log.debug('keys in __context__: {}'.format(list(__context__)))
hubblestack.utils.stdrec.__grains__ = __grains__
hubblestack.utils.stdrec.__opts__ = __opts__
hubblestack.hec.opt.__grains__ = __grains__
hubblestack.hec.opt.__mods__ = __mods__
hubblestack.hec.opt.__opts__ = __opts__
hubblestack.log.splunk.__grains__ = __grains__
hubblestack.log.splunk.__mods__ = __mods__
hubblestack.log.splunk.__opts__ = __opts__
hubblestack.status.__opts__ = __opts__
hubblestack.status.__mods__ = __mods__
hubblestack.utils.signing.__opts__ = __opts__
hubblestack.utils.signing.__mods__ = __mods__
hubblestack.module_runner.runner.__mods__ = __mods__
hubblestack.module_runner.runner.__grains__ = __grains__
hubblestack.module_runner.runner.__opts__ = __opts__
hubblestack.module_runner.audit_runner.__mods__ = __mods__
hubblestack.module_runner.audit_runner.__grains__ = __grains__
hubblestack.module_runner.audit_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__mods__ = __mods__
hubblestack.module_runner.fdg_runner.__grains__ = __grains__
hubblestack.module_runner.fdg_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__returners__ = __returners__
hubblestack.utils.signing.__mods__ = __mods__
HSS.start_sigusr1_signal_handler()
hubblestack.log.refresh_handler_std_info()
clear_selective_context()
if not initial and __mods__["config.get"]("splunklogging", False):
hubblestack.log.emit_to_splunk(__grains__, "INFO",
"hubblestack.grains_report")