def dumpster_fire(cls, *_a, **_kw):
""" dump the status.json file to cachedir
Location and filename can be adjusted with the cachedir and
hubble:status:dumpster options (see above).
"""
try:
if __salt__['config.get']('splunklogging', False):
# lazy load to avoid circular import
import hubblestack.log
hubblestack.log.emit_to_splunk('Signal {0} detected'.format(signal.SIGUSR1),
'INFO',
'hubblestack.signals')
finally:
dumpster = get_hubble_status_opt('dumpster') or 'status.json'
if not dumpster.startswith('/'):
cachedir = get_hubble_or_salt_opt('cachedir') or '/tmp'
dumpster = os.path.join(cachedir, dumpster)
try:
with open(dumpster, 'w') as status_file:
status_file.write(cls.as_json())
status_file.write('\n')
log.info("wrote HubbleStatus to %s", dumpster)
except Exception:
log.exception("ignoring exception during dumpster fire")
def emit_to_syslog(grains_to_emit):
'''
Emit grains and their values to syslog
'''
try:
# Avoid a syslog line to be longer than 1024 characters
# Build syslog message
syslog_list = []
syslog_list.append('hubble_syslog_message:')
for grain in grains_to_emit:
if grain in __grains__:
if bool(__grains__[grain]) and isinstance(
__grains__[grain], dict):
for key, value in __grains__[grain].iteritems():
syslog_list.append('{0}={1}'.format(key, value))
else:
syslog_list.append('{0}={1}'.format(
grain, __grains__[grain]))
syslog_message = ' '.join(syslog_list)
log.info('Emitting some grains to syslog')
syslog.openlog(logoption=syslog.LOG_PID)
syslog.syslog(syslog_message)
except Exception as e:
log.exception(
'An exception occurred on emitting a message to syslog: {0}'.
format(e))
def _setup_cached_uuid():
""" Get the cached uuid and cached system uui path, read the files
and remove the cached uuid """
# Check for a cloned system with existing hubble_uuid
def _get_uuid_from_system():
query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv'
# Prefer our /opt/osquery/osqueryi if present
osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi',
'/usr/bin/osqueryi')
for path in osqueryipaths:
if hubblestack.utils.path.which(path):
live_uuid = hubblestack.modules.cmdmod.run_stdout(
'{0} {1}'.format(path, query), output_loglevel='quiet')
live_uuid = str(live_uuid).upper()
if len(live_uuid) == 36:
return live_uuid
return None
# If osquery isn't available, attempt to get uuid from /sys path (linux only)
try:
with open('/sys/devices/virtual/dmi/id/product_uuid',
'r') as product_uuid_file:
file_uuid = product_uuid_file.read()
file_uuid = str(file_uuid).upper()
if len(file_uuid) == 36:
return file_uuid
return None
except Exception:
return None
cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']),
'hubble_cached_uuid')
cached_system_uuid_path = os.path.join(
os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid')
try:
if os.path.isfile(cached_uuid_path) and os.path.isfile(
cached_system_uuid_path):
with open(cached_uuid_path, 'r') as cached_uuid_file, \
open(cached_system_uuid_path, 'r') as cached_system_uuid_file:
cached_uuid = cached_uuid_file.read()
cached_system_uuid = cached_system_uuid_file.read()
if cached_uuid != cached_system_uuid:
live_uuid = _get_uuid_from_system()
if live_uuid != cached_system_uuid:
log.error(
"potentially cloned system detected: System_uuid grain "
"previously saved on disk doesn't match live system value.\n"
"Resettig cached hubble_uuid value.")
os.remove(cached_uuid_path)
except Exception:
log.exception(
"Problem opening cache files while checking for previously cloned system"
)
def _update_fileserver(file_client):
""" Update the filserver and the last_fc_update time """
try:
file_client.channel.fs.update()
last_fc_update = time.time()
except Exception:
retry = __opts__.get('fileserver_retry_rate', 900)
last_fc_update += retry
log.exception('Exception thrown trying to update fileclient. '
'Trying again in %s seconds.', retry)
return last_fc_update
def main():
"""
Run the main hubble loop
"""
# Initial fileclient setup
_clear_gitfs_locks()
# Setup fileclient
log.info('Setting up the fileclient/fileserver')
retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
count = 0
while True:
try:
file_client = hubblestack.fileclient.get_file_client(__opts__)
file_client.channel.fs.update()
last_fc_update = time.time()
break
except Exception:
if (retry_count is None
or count < retry_count) and not __opts__['function']:
log.exception(
'Exception thrown trying to setup fileclient. '
'Trying again in %s seconds.', retry_time)
count += 1
time.sleep(retry_time)
continue
else:
log.exception(
'Exception thrown trying to setup fileclient. Exiting.')
sys.exit(1)
# Check for single function run
if __opts__['function']:
run_function()
sys.exit(0)
last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']
log.info('Starting main loop')
pidfile_count = 0
# pidfile_refresh in seconds, our scheduler deals in half-seconds
pidfile_refresh = int(__opts__.get('pidfile_refresh', 60)) * 2
while True:
# Check if fileserver needs update
if time.time(
) - last_fc_update >= __opts__['fileserver_update_frequency']:
last_fc_update = _update_fileserver(file_client)
pidfile_count += 1
if __opts__['daemonize'] and pidfile_count > pidfile_refresh:
pidfile_count = 0
create_pidfile()
if time.time(
) - last_grains_refresh >= __opts__['grains_refresh_frequency']:
last_grains_refresh = _emit_and_refresh_grains()
try:
log.debug('Executing schedule')
sf_count = schedule()
except Exception as exc:
log.exception('Error executing schedule: %s', exc)
if isinstance(exc, KeyboardInterrupt):
raise exc
time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))
def emit_to_syslog(grains_to_emit):
"""
Emit grains and their values to syslog
"""
try:
# Avoid a syslog line to be longer than 1024 characters
# Build syslog message
syslog_list = ["hubble_syslog_message:"]
for grain in grains_to_emit:
if grain in __grains__:
if bool(__grains__[grain]) and isinstance(
__grains__[grain], dict):
for key, value in __grains__[grain].items():
syslog_list.append("{0}={1}".format(key, value))
else:
syslog_list.append("{0}={1}".format(
grain, __grains__[grain]))
syslog_message = " ".join(syslog_list)
log.info("Emitting some grains to syslog")
syslog.openlog(logoption=syslog.LOG_PID)
syslog.syslog(syslog_message)
except Exception as exc:
log.exception(
"An exception occurred on emitting a message to syslog: %s", exc)
def main():
'''
Run the main hubble loop
'''
# Initial fileclient setup
# Clear old locks
if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__[
'fileserver_backend']:
git_objects = [
salt.utils.gitfs.GitFS(
__opts__,
__opts__['gitfs_remotes'],
per_remote_overrides=salt.fileserver.gitfs.
PER_REMOTE_OVERRIDES,
per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY)
]
ret = {}
for obj in git_objects:
lock_type = 'update'
cleared, errors = salt.fileserver.clear_lock(obj.clear_lock,
'gitfs',
remote=None,
lock_type=lock_type)
if cleared:
ret.setdefault('cleared', []).extend(cleared)
if errors:
ret.setdefault('errors', []).extend(errors)
if ret:
log.info('One or more gitfs locks were removed: {0}'.format(ret))
# Setup fileclient
log.info('Setting up the fileclient/fileserver')
# Set up fileclient
retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
count = 0
while True:
try:
fc = salt.fileclient.get_file_client(__opts__)
fc.channel.fs.update()
last_fc_update = time.time()
break
except Exception as exc:
if (retry_count is None
or count < retry_count) and not __opts__['function']:
log.exception(
'Exception thrown trying to setup fileclient. '
'Trying again in {0} seconds.'.format(retry_time))
count += 1
time.sleep(retry_time)
continue
else:
log.exception(
'Exception thrown trying to setup fileclient. Exiting.')
sys.exit(1)
# Check for single function run
if __opts__['function']:
run_function()
sys.exit(0)
last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']
log.info('Starting main loop')
pidfile_count = 0
# pidfile_refresh in seconds, our scheduler deals in half-seconds
pidfile_refresh = int(__opts__.get('pidfile_refresh', 60)) * 2
while True:
# Check if fileserver needs update
if time.time(
) - last_fc_update >= __opts__['fileserver_update_frequency']:
try:
fc.channel.fs.update()
last_fc_update = time.time()
except Exception as exc:
retry = __opts__.get('fileserver_retry_rate', 900)
last_fc_update += retry
log.exception('Exception thrown trying to update fileclient. '
'Trying again in {0} seconds.'.format(retry))
pidfile_count += 1
if __opts__['daemonize'] and pidfile_count > pidfile_refresh:
pidfile_count = 0
create_pidfile()
if time.time(
) - last_grains_refresh >= __opts__['grains_refresh_frequency']:
log.info('Refreshing grains')
refresh_grains()
last_grains_refresh = time.time()
# Emit syslog at grains refresh frequency
if not (salt.utils.platform.is_windows()) and __opts__.get(
'emit_grains_to_syslog', True):
default_grains_to_emit = [
'system_uuid', 'hubble_uuid', 'session_uuid', 'machine_id',
'splunkindex', 'cloud_details', 'hubble_version',
'localhost', 'fqdn'
]
grains_to_emit = []
grains_to_emit.extend(
__opts__.get('emit_grains_to_syslog_list',
default_grains_to_emit))
emit_to_syslog(grains_to_emit)
try:
log.debug('Executing schedule')
sf_count = schedule()
if sf_count > 0:
log.debug('Executed %d schedule item(s)', sf_count)
hubblestack.log.workaround_salt_log_handler_queues()
except Exception as e:
log.exception('Error executing schedule: {0}'.format(e))
if isinstance(e, KeyboardInterrupt):
raise e
time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))
def load_config():
'''
Load the config from configfile and load into imported salt modules
'''
# Parse arguments
parsed_args = parse_args()
# Let's find out the path of this module
if 'SETUP_DIRNAME' in globals():
# This is from the exec() call in Salt's setup.py
this_file = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py') # pylint: disable=E0602
else:
this_file = __file__
install_dir = os.path.dirname(os.path.realpath(this_file))
# Load unique data for Windows or Linux
if salt.utils.platform.is_windows():
if parsed_args.get('configfile') is None:
parsed_args[
'configfile'] = 'C:\\Program Files (x86)\\Hubble\\etc\\hubble\\hubble.conf'
salt.config.DEFAULT_MINION_OPTS[
'cachedir'] = 'C:\\Program Files (x86)\\hubble\\var\\cache'
salt.config.DEFAULT_MINION_OPTS[
'pidfile'] = 'C:\\Program Files (x86)\\hubble\\var\\run\\hubble.pid'
salt.config.DEFAULT_MINION_OPTS[
'log_file'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble.log'
salt.config.DEFAULT_MINION_OPTS[
'osquery_dbpath'] = 'C:\\Program Files (x86)\\hubble\\var\\hubble_osquery_db'
salt.config.DEFAULT_MINION_OPTS[
'osquerylogpath'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery'
salt.config.DEFAULT_MINION_OPTS['osquerylog_backupdir'] = \
'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery\\backuplogs'
else:
if parsed_args.get('configfile') is None:
parsed_args['configfile'] = '/etc/hubble/hubble'
salt.config.DEFAULT_MINION_OPTS['cachedir'] = '/var/cache/hubble'
salt.config.DEFAULT_MINION_OPTS['pidfile'] = '/var/run/hubble.pid'
salt.config.DEFAULT_MINION_OPTS['log_file'] = '/var/log/hubble'
salt.config.DEFAULT_MINION_OPTS[
'osquery_dbpath'] = '/var/cache/hubble/osquery'
salt.config.DEFAULT_MINION_OPTS[
'osquerylogpath'] = '/var/log/hubble_osquery'
salt.config.DEFAULT_MINION_OPTS[
'osquerylog_backupdir'] = '/var/log/hubble_osquery/backuplogs'
salt.config.DEFAULT_MINION_OPTS['file_roots'] = {'base': []}
salt.config.DEFAULT_MINION_OPTS['log_level'] = 'error'
salt.config.DEFAULT_MINION_OPTS['file_client'] = 'local'
salt.config.DEFAULT_MINION_OPTS[
'fileserver_update_frequency'] = 43200 # 12 hours
salt.config.DEFAULT_MINION_OPTS[
'grains_refresh_frequency'] = 3600 # 1 hour
salt.config.DEFAULT_MINION_OPTS['scheduler_sleep_frequency'] = 0.5
salt.config.DEFAULT_MINION_OPTS['default_include'] = 'hubble.d/*.conf'
salt.config.DEFAULT_MINION_OPTS['logfile_maxbytes'] = 100000000 # 100MB
salt.config.DEFAULT_MINION_OPTS[
'logfile_backups'] = 1 # maximum rotated logs
salt.config.DEFAULT_MINION_OPTS[
'delete_inaccessible_azure_containers'] = False
salt.config.DEFAULT_MINION_OPTS[
'enable_globbing_in_nebula_masking'] = False # Globbing will not be supported in nebula masking
salt.config.DEFAULT_MINION_OPTS[
'osquery_logfile_maxbytes'] = 50000000 # 50MB
salt.config.DEFAULT_MINION_OPTS[
'osquery_logfile_maxbytes_toparse'] = 100000000 #100MB
salt.config.DEFAULT_MINION_OPTS['osquery_backuplogs_count'] = 2
global __opts__
__opts__ = salt.config.minion_config(parsed_args.get('configfile'))
__opts__.update(parsed_args)
__opts__['conf_file'] = parsed_args.get('configfile')
__opts__['install_dir'] = install_dir
if __opts__['version']:
print(__version__)
clean_up_process(None, None)
sys.exit(0)
if __opts__['buildinfo']:
try:
from hubblestack import __buildinfo__
except ImportError:
__buildinfo__ = 'NOT SET'
print(__buildinfo__)
clean_up_process(None, None)
sys.exit(0)
scan_proc = __opts__.get('scan_proc', False)
if __opts__['daemonize']:
# before becoming a daemon, check for other procs and possibly send
# them a signal 15 (otherwise refuse to run)
if not __opts__.get('ignore_running', False):
check_pidfile(kill_other=True, scan_proc=scan_proc)
salt.utils.daemonize()
create_pidfile()
elif not __opts__['function'] and not __opts__['version'] and not __opts__[
'buildinfo']:
# check the pidfile and possibly refuse to run
# (assuming this isn't a single function call)
if not __opts__.get('ignore_running', False):
check_pidfile(kill_other=False, scan_proc=scan_proc)
signal.signal(signal.SIGTERM, clean_up_process)
signal.signal(signal.SIGINT, clean_up_process)
signal.signal(signal.SIGABRT, clean_up_process)
signal.signal(signal.SIGFPE, clean_up_process)
signal.signal(signal.SIGILL, clean_up_process)
signal.signal(signal.SIGSEGV, clean_up_process)
if not salt.utils.platform.is_windows():
signal.signal(signal.SIGHUP, clean_up_process)
signal.signal(signal.SIGQUIT, clean_up_process)
# Optional sleep to wait for network
time.sleep(int(__opts__.get('startup_sleep', 0)))
# Convert -vvv to log level
if __opts__['log_level'] is None:
# Default to 'error'
__opts__['log_level'] = 'error'
# Default to more verbose if we're daemonizing
if __opts__['daemonize']:
__opts__['log_level'] = 'info'
# Handle the explicit -vvv settings
if __opts__['verbose'] == 1:
__opts__['log_level'] = 'warning'
elif __opts__['verbose'] == 2:
__opts__['log_level'] = 'info'
elif __opts__['verbose'] >= 3:
__opts__['log_level'] = 'debug'
# Setup module/grain/returner dirs
module_dirs = __opts__.get('module_dirs', [])
module_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'modules'))
__opts__['module_dirs'] = module_dirs
grains_dirs = __opts__.get('grains_dirs', [])
grains_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'grains'))
__opts__['grains_dirs'] = grains_dirs
returner_dirs = __opts__.get('returner_dirs', [])
returner_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'returners'))
__opts__['returner_dirs'] = returner_dirs
fileserver_dirs = __opts__.get('fileserver_dirs', [])
fileserver_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'fileserver'))
__opts__['fileserver_dirs'] = fileserver_dirs
utils_dirs = __opts__.get('utils_dirs', [])
utils_dirs.append(
os.path.join(os.path.dirname(__file__), 'extmods', 'utils'))
__opts__['utils_dirs'] = utils_dirs
fdg_dirs = __opts__.get('fdg_dirs', [])
fdg_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'fdg'))
__opts__['fdg_dirs'] = fdg_dirs
__opts__['file_roots']['base'].insert(
0, os.path.join(os.path.dirname(__file__), 'files'))
if 'roots' not in __opts__['fileserver_backend']:
__opts__['fileserver_backend'].append('roots')
# Disable all of salt's boto modules, they give nothing but trouble to the loader
disable_modules = __opts__.get('disable_modules', [])
disable_modules.extend([
'boto3_elasticache',
'boto3_route53',
'boto3_sns',
'boto_apigateway',
'boto_asg',
'boto_cfn',
'boto_cloudfront',
'boto_cloudtrail',
'boto_cloudwatch_event',
'boto_cloudwatch',
'boto_cognitoidentity',
'boto_datapipeline',
'boto_dynamodb',
'boto_ec2',
'boto_efs',
'boto_elasticache',
'boto_elasticsearch_domain',
'boto_elb',
'boto_elbv2',
'boto_iam',
'boto_iot',
'boto_kinesis',
'boto_kms',
'boto_lambda',
'boto_rds',
'boto_route53',
'boto_s3_bucket',
'boto_s3',
'boto_secgroup',
'boto_sns',
'boto_sqs',
'boto_ssm',
'boto_vpc',
])
__opts__['disable_modules'] = disable_modules
# Console logging is probably the same, but can be different
console_logging_opts = {
'log_level':
__opts__.get('console_log_level', __opts__['log_level']),
'log_format':
__opts__.get('console_log_format',
'%(asctime)s [%(levelname)-5s] %(message)s'),
'date_format':
__opts__.get('console_log_date_format', '%H:%M:%S'),
}
file_logging_opts = {
'log_file':
__opts__.get('log_file', '/var/log/hubble'),
'log_level':
__opts__['log_level'],
'log_format':
__opts__.get(
'log_format',
'%(asctime)s,%(msecs)03d [%(levelname)-5s] [%(name)s:%(lineno)d] %(message)s'
),
'date_format':
__opts__.get('log_date_format', '%Y-%m-%d %H:%M:%S'),
'max_bytes':
__opts__.get('logfile_maxbytes', 100000000),
'backup_count':
__opts__.get('logfile_backups', 1),
}
# Setup logging
hubblestack.log.setup_console_logger(**console_logging_opts)
hubblestack.log.setup_file_logger(**file_logging_opts)
# 384 is 0o600 permissions, written without octal for python 2/3 compat
os.chmod(__opts__['log_file'], 384)
os.chmod(parsed_args.get('configfile'), 384)
# Check for a cloned system with existing hubble_uuid
def _get_uuid_from_system():
query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv'
# Prefer our /opt/osquery/osqueryi if present
osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi',
'/usr/bin/osqueryi')
for path in osqueryipaths:
if salt.utils.path.which(path):
live_uuid = salt.modules.cmdmod.run_stdout(
'{0} {1}'.format(path, query), output_loglevel='quiet')
live_uuid = str(live_uuid).upper()
if len(live_uuid) == 36:
return live_uuid
else:
return None
# If osquery isn't available, attempt to get uuid from /sys path (linux only)
try:
with open('/sys/devices/virtual/dmi/id/product_uuid', 'r') as f:
file_uuid = f.read()
file_uuid = str(file_uuid).upper()
if len(file_uuid) == 36:
return file_uuid
else:
return None
except Exception:
return None
cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']),
'hubble_cached_uuid')
cached_system_uuid_path = os.path.join(
os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid')
try:
if os.path.isfile(cached_uuid_path) and os.path.isfile(
cached_system_uuid_path):
with open(cached_uuid_path,
'r') as f, open(cached_system_uuid_path, 'r') as g:
cached_uuid = f.read()
cached_system_uuid = g.read()
if cached_uuid != cached_system_uuid:
live_uuid = _get_uuid_from_system()
if live_uuid != cached_system_uuid:
log.error(
"potentially cloned system detected: System_uuid grain "
"previously saved on disk doesn't match live system value.\n"
"Resettig cached hubble_uuid value.")
os.remove(cached_uuid_path)
except Exception:
log.exception(
"Problem opening cache files while checking for previously cloned system"
)
refresh_grains(initial=True)
if __salt__['config.get']('splunklogging', False):
hubblestack.log.setup_splunk_logger()
hubblestack.log.emit_to_splunk(__grains__,
'INFO',
'hubblestack.grains_report',
remove_sensitive_logs=True)
