def dumpster_fire(cls, *_a, **_kw):
""" dump the status.json file to cachedir
Location and filename can be adjusted with the cachedir and
hubble:status:dumpster options (see above).
"""
try:
if __salt__['config.get']('splunklogging', False):
# lazy load to avoid circular import
import hubblestack.log
hubblestack.log.emit_to_splunk('Signal {0} detected'.format(signal.SIGUSR1),
'INFO',
'hubblestack.signals')
finally:
dumpster = get_hubble_status_opt('dumpster') or 'status.json'
if not dumpster.startswith('/'):
cachedir = get_hubble_or_salt_opt('cachedir') or '/tmp'
dumpster = os.path.join(cachedir, dumpster)
try:
with open(dumpster, 'w') as status_file:
status_file.write(cls.as_json())
status_file.write('\n')
log.info("wrote HubbleStatus to %s", dumpster)
except Exception:
log.exception("ignoring exception during dumpster fire")
def emit_to_syslog(grains_to_emit):
'''
Emit grains and their values to syslog
'''
try:
# Avoid a syslog line to be longer than 1024 characters
# Build syslog message
syslog_list = []
syslog_list.append('hubble_syslog_message:')
for grain in grains_to_emit:
if grain in __grains__:
if bool(__grains__[grain]) and isinstance(
__grains__[grain], dict):
for key, value in __grains__[grain].iteritems():
syslog_list.append('{0}={1}'.format(key, value))
else:
syslog_list.append('{0}={1}'.format(
grain, __grains__[grain]))
syslog_message = ' '.join(syslog_list)
log.info('Emitting some grains to syslog')
syslog.openlog(logoption=syslog.LOG_PID)
syslog.syslog(syslog_message)
except Exception as e:
log.exception(
'An exception occurred on emitting a message to syslog: {0}'.
format(e))
def _clear_gitfs_locks():
"""Clear old locks and log the changes"""
# Clear old locks
if "gitfs" in __opts__["fileserver_backend"] or "git" in __opts__[
"fileserver_backend"]:
git_objects = [
hubblestack.utils.gitfs.GitFS(
__opts__,
__opts__["gitfs_remotes"],
per_remote_overrides=hubblestack.fileserver.gitfs.
PER_REMOTE_OVERRIDES,
per_remote_only=hubblestack.fileserver.gitfs.PER_REMOTE_ONLY,
)
]
ret = {}
for obj in git_objects:
lock_type = "update"
cleared, errors = hubblestack.fileserver.clear_lock(
obj.clear_lock, "gitfs", remote=None, lock_type=lock_type)
if cleared:
ret.setdefault("cleared", []).extend(cleared)
if errors:
ret.setdefault("errors", []).extend(errors)
if ret:
log.info("One or more gitfs locks were removed: %s", ret)
def main():
"""
Run the main hubble loop
"""
# Initial fileclient setup
_clear_gitfs_locks()
# Setup fileclient
log.info('Setting up the fileclient/fileserver')
retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
count = 0
while True:
try:
file_client = hubblestack.fileclient.get_file_client(__opts__)
file_client.channel.fs.update()
last_fc_update = time.time()
break
except Exception:
if (retry_count is None
or count < retry_count) and not __opts__['function']:
log.exception(
'Exception thrown trying to setup fileclient. '
'Trying again in %s seconds.', retry_time)
count += 1
time.sleep(retry_time)
continue
else:
log.exception(
'Exception thrown trying to setup fileclient. Exiting.')
sys.exit(1)
# Check for single function run
if __opts__['function']:
run_function()
sys.exit(0)
last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']
log.info('Starting main loop')
pidfile_count = 0
# pidfile_refresh in seconds, our scheduler deals in half-seconds
pidfile_refresh = int(__opts__.get('pidfile_refresh', 60)) * 2
while True:
# Check if fileserver needs update
if time.time(
) - last_fc_update >= __opts__['fileserver_update_frequency']:
last_fc_update = _update_fileserver(file_client)
pidfile_count += 1
if __opts__['daemonize'] and pidfile_count > pidfile_refresh:
pidfile_count = 0
create_pidfile()
if time.time(
) - last_grains_refresh >= __opts__['grains_refresh_frequency']:
last_grains_refresh = _emit_and_refresh_grains()
try:
log.debug('Executing schedule')
sf_count = schedule()
except Exception as exc:
log.exception('Error executing schedule: %s', exc)
if isinstance(exc, KeyboardInterrupt):
raise exc
time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))
def start_sigusr1_signal_handler(cls):
""" start the signal.SIGUSR1 handler (dumps status to
/var/cache/hubble/status.json or whatever is specified in
cachedir + hubble:status:dumpster configs)
"""
if not cls._signaled:
cls._signaled = True
if not hasattr(signal, 'SIGUSR1'):
# TODO: invent invocation that works in windows instead of just complaining
log.info("signal package lacks SIGUSR1, skipping SIGUSR1 status.json handler setup")
return
signal.signal(signal.SIGUSR1, cls.dumpster_fire)
def run_function():
'''
Run a single function requested by the user
'''
# Parse the args
args = []
kwargs = {}
for arg in __opts__['args']:
if '=' in arg:
kwarg, _, value = arg.partition('=')
kwargs[kwarg] = value
else:
args.append(arg)
log.debug('Parsed args: {0} | Parsed kwargs: {1}'.format(args, kwargs))
log.info('Executing user-requested function {0}'.format(
__opts__['function']))
try:
ret = __salt__[__opts__['function']](*args, **kwargs)
except KeyError:
log.error('Function {0} is not available, or not valid.'.format(
__opts__['function']))
sys.exit(1)
if __opts__['return']:
returner = '{0}.returner'.format(__opts__['return'])
if returner not in __returners__:
log.error('Could not find {0} returner.'.format(returner))
else:
log.info('Returning job data to {0}'.format(returner))
returner_ret = {
'id': __grains__['id'],
'jid': salt.utils.jid.gen_jid(__opts__),
'fun': __opts__['function'],
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret,
'retry': False
}
if __opts__.get('returner_retry', False):
returner_ret['retry'] = True
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__['json_print']:
print(json.dumps(ret))
else:
if not __opts__['no_pprint']:
pprint.pprint(ret)
else:
print(ret)
def _emit_and_refresh_grains():
""" When the grains refresh frequency has expired, refresh grains and emit to syslog """
log.info('Refreshing grains')
refresh_grains()
last_grains_refresh = time.time()
# Emit syslog at grains refresh frequency
if not (salt.utils.platform.is_windows()) and \
__opts__.get('emit_grains_to_syslog', True):
default_grains_to_emit = ['system_uuid', 'hubble_uuid', 'session_uuid',
'machine_id', 'splunkindex', 'cloud_details',
'hubble_version', 'localhost', 'fqdn']
grains_to_emit = []
grains_to_emit.extend(
__opts__.get('emit_grains_to_syslog_list', default_grains_to_emit))
emit_to_syslog(grains_to_emit)
return last_grains_refresh
def run_function():
"""
Run a single function requested by the user
"""
# Parse the args
args = []
kwargs = {}
for arg in __opts__['args']:
if '=' in arg:
kwarg, _, value = arg.partition('=')
kwargs[kwarg] = value
else:
args.append(arg)
log.debug('Parsed args: %s | Parsed kwargs: %s', args, kwargs)
log.info('Executing user-requested function %s', __opts__['function'])
mod_fun = __mods__.get(__opts__['function'])
if not mod_fun or not callable(mod_fun):
log.error('Function %s is not available, or not valid.',
__opts__['function'])
sys.exit(1)
ret = mod_fun(*args, **kwargs)
if __opts__['return']:
returner = '{0}.returner'.format(__opts__['return'])
if returner not in __returners__:
log.error('Could not find %s returner.', returner)
else:
log.info('Returning job data to %s', returner)
returner_ret = {
'id': __grains__['id'],
'jid': hubblestack.utils.jid.gen_jid(__opts__),
'fun': __opts__['function'],
'fun_args': args + ([kwargs] if kwargs else []),
'return': ret
}
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__['json_print']:
print(json.dumps(ret))
else:
if not __opts__['no_pprint']:
pprint.pprint(ret)
else:
print(ret)
def run_function():
"""
Run a single function requested by the user
"""
# Parse the args
args = []
kwargs = {}
for arg in __opts__["args"]:
if "=" in arg:
kwarg, _, value = arg.partition("=")
kwargs[kwarg] = value
else:
args.append(arg)
log.debug("Parsed args: %s | Parsed kwargs: %s", args, kwargs)
log.info("Executing user-requested function %s", __opts__["function"])
mod_fun = __mods__.get(__opts__["function"])
if not mod_fun or not callable(mod_fun):
log.error("Function %s is not available, or not valid.",
__opts__["function"])
sys.exit(1)
ret = mod_fun(*args, **kwargs)
if __opts__["return"]:
returner = "{0}.returner".format(__opts__["return"])
if returner not in __returners__:
log.error("Could not find %s returner.", returner)
else:
log.info("Returning job data to %s", returner)
returner_ret = {
"id": __grains__["id"],
"jid": hubblestack.utils.jid.gen_jid(__opts__),
"fun": __opts__["function"],
"fun_args": args + ([kwargs] if kwargs else []),
"return": ret,
}
__returners__[returner](returner_ret)
# TODO instantiate the salt outputter system?
if __opts__["json_print"]:
print(json.dumps(ret))
else:
if not __opts__["no_pprint"]:
pprint.pprint(ret)
else:
print(ret)
def _clear_gitfs_locks():
""" Clear old locks and log the changes """
# Clear old locks
if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__['fileserver_backend']:
git_objects = [
salt.utils.gitfs.GitFS(
__opts__,
__opts__['gitfs_remotes'],
per_remote_overrides=salt.fileserver.gitfs.PER_REMOTE_OVERRIDES,
per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY)]
ret = {}
for obj in git_objects:
lock_type = 'update'
cleared, errors = salt.fileserver.clear_lock(obj.clear_lock, 'gitfs', remote=None,
lock_type=lock_type)
if cleared:
ret.setdefault('cleared', []).extend(cleared)
if errors:
ret.setdefault('errors', []).extend(errors)
if ret:
log.info('One or more gitfs locks were removed: %s', ret)
def emit_to_syslog(grains_to_emit):
"""
Emit grains and their values to syslog
"""
try:
# Avoid a syslog line to be longer than 1024 characters
# Build syslog message
syslog_list = ["hubble_syslog_message:"]
for grain in grains_to_emit:
if grain in __grains__:
if bool(__grains__[grain]) and isinstance(
__grains__[grain], dict):
for key, value in __grains__[grain].items():
syslog_list.append("{0}={1}".format(key, value))
else:
syslog_list.append("{0}={1}".format(
grain, __grains__[grain]))
syslog_message = " ".join(syslog_list)
log.info("Emitting some grains to syslog")
syslog.openlog(logoption=syslog.LOG_PID)
syslog.syslog(syslog_message)
except Exception as exc:
log.exception(
"An exception occurred on emitting a message to syslog: %s", exc)
def _emit_and_refresh_grains():
"""When the grains refresh frequency has expired, refresh grains and emit to syslog"""
log.info("Refreshing grains")
refresh_grains()
last_grains_refresh = time.time()
# Emit syslog at grains refresh frequency
if not (hubblestack.utils.platform.is_windows()) and __opts__.get(
"emit_grains_to_syslog", True):
default_grains_to_emit = [
"system_uuid",
"hubble_uuid",
"session_uuid",
"machine_id",
"splunkindex",
"cloud_details",
"hubble_version",
"localhost",
"fqdn",
]
grains_to_emit = []
grains_to_emit.extend(
__opts__.get("emit_grains_to_syslog_list", default_grains_to_emit))
emit_to_syslog(grains_to_emit)
return last_grains_refresh
def refresh_grains(initial=False):
"""
Refresh the grains, pillar, utils, modules, and returners
"""
global __opts__
global __grains__
global __utils__
global __mods__
global __pillar__
global __returners__
global __context__
persist, old_grains = {}, {}
if not initial:
old_grains = copy.deepcopy(__grains__)
for grain in __opts__.get('grains_persist', []):
if grain in __grains__:
persist[grain] = __grains__[grain]
# Hardcode these core grains as persisting
persist = {
grain: __grains__[grain]
for grain in ['hubble_version', 'buildinfo'] if grain in __grains__
}
if initial:
__context__ = {}
if 'grains' in __opts__:
__opts__.pop('grains')
if 'pillar' in __opts__:
__opts__.pop('pillar')
__grains__ = hubblestack.loader.grains(__opts__)
__grains__.update(persist)
__grains__['session_uuid'] = SESSION_UUID
# This was a weird one. In older versions of hubble the version and
# buildinfo were not persisted automatically which means that if you
# installed a new version without restarting hubble, grains refresh could
# cause that old daemon to report grains as if it were the new version.
# Now if this hubble_marker_3 grain is present you know you can trust the
# hubble_version and buildinfo.
__grains__['hubble_marker_3'] = True
old_grains.update(__grains__)
__grains__ = old_grains
# Check for default gateway and fall back if necessary
if __grains__.get(
'ip_gw',
None) is False and 'fallback_fileserver_backend' in __opts__:
log.info(
'No default gateway detected; using fallback_fileserver_backend.')
__opts__['fileserver_backend'] = __opts__[
'fallback_fileserver_backend']
__opts__['hubble_uuid'] = __grains__.get('hubble_uuid', None)
__opts__['system_uuid'] = __grains__.get('system_uuid', None)
__pillar__ = {}
__opts__['grains'] = __grains__
__opts__['pillar'] = __pillar__
__utils__ = hubblestack.loader.utils(__opts__)
__mods__ = hubblestack.loader.modules(__opts__,
utils=__utils__,
context=__context__)
__returners__ = hubblestack.loader.returners(__opts__, __mods__)
# the only things that turn up in here (and that get preserved)
# are pulsar.queue, pulsar.notifier and cp.fileclient_###########
# log.debug('keys in __context__: {}'.format(list(__context__)))
hubblestack.utils.stdrec.__grains__ = __grains__
hubblestack.utils.stdrec.__opts__ = __opts__
hubblestack.hec.opt.__grains__ = __grains__
hubblestack.hec.opt.__mods__ = __mods__
hubblestack.hec.opt.__opts__ = __opts__
hubblestack.log.splunk.__grains__ = __grains__
hubblestack.log.splunk.__mods__ = __mods__
hubblestack.log.splunk.__opts__ = __opts__
hubblestack.status.__opts__ = __opts__
hubblestack.status.__mods__ = __mods__
hubblestack.utils.signing.__opts__ = __opts__
hubblestack.utils.signing.__mods__ = __mods__
hubblestack.module_runner.runner.__mods__ = __mods__
hubblestack.module_runner.runner.__grains__ = __grains__
hubblestack.module_runner.runner.__opts__ = __opts__
hubblestack.module_runner.audit_runner.__mods__ = __mods__
hubblestack.module_runner.audit_runner.__grains__ = __grains__
hubblestack.module_runner.audit_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__mods__ = __mods__
hubblestack.module_runner.fdg_runner.__grains__ = __grains__
hubblestack.module_runner.fdg_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__returners__ = __returners__
hubblestack.utils.signing.__mods__ = __mods__
HSS.start_sigusr1_signal_handler()
hubblestack.log.refresh_handler_std_info()
clear_selective_context()
if not initial and __mods__['config.get']('splunklogging', False):
hubblestack.log.emit_to_splunk(__grains__, 'INFO',
'hubblestack.grains_report')
def main():
'''
Run the main hubble loop
'''
# Initial fileclient setup
# Clear old locks
if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__[
'fileserver_backend']:
git_objects = [
salt.utils.gitfs.GitFS(
__opts__,
__opts__['gitfs_remotes'],
per_remote_overrides=salt.fileserver.gitfs.
PER_REMOTE_OVERRIDES,
per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY)
]
ret = {}
for obj in git_objects:
lock_type = 'update'
cleared, errors = salt.fileserver.clear_lock(obj.clear_lock,
'gitfs',
remote=None,
lock_type=lock_type)
if cleared:
ret.setdefault('cleared', []).extend(cleared)
if errors:
ret.setdefault('errors', []).extend(errors)
if ret:
log.info('One or more gitfs locks were removed: {0}'.format(ret))
# Setup fileclient
log.info('Setting up the fileclient/fileserver')
# Set up fileclient
retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
count = 0
while True:
try:
fc = salt.fileclient.get_file_client(__opts__)
fc.channel.fs.update()
last_fc_update = time.time()
break
except Exception as exc:
if (retry_count is None
or count < retry_count) and not __opts__['function']:
log.exception(
'Exception thrown trying to setup fileclient. '
'Trying again in {0} seconds.'.format(retry_time))
count += 1
time.sleep(retry_time)
continue
else:
log.exception(
'Exception thrown trying to setup fileclient. Exiting.')
sys.exit(1)
# Check for single function run
if __opts__['function']:
run_function()
sys.exit(0)
last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']
log.info('Starting main loop')
pidfile_count = 0
# pidfile_refresh in seconds, our scheduler deals in half-seconds
pidfile_refresh = int(__opts__.get('pidfile_refresh', 60)) * 2
while True:
# Check if fileserver needs update
if time.time(
) - last_fc_update >= __opts__['fileserver_update_frequency']:
try:
fc.channel.fs.update()
last_fc_update = time.time()
except Exception as exc:
retry = __opts__.get('fileserver_retry_rate', 900)
last_fc_update += retry
log.exception('Exception thrown trying to update fileclient. '
'Trying again in {0} seconds.'.format(retry))
pidfile_count += 1
if __opts__['daemonize'] and pidfile_count > pidfile_refresh:
pidfile_count = 0
create_pidfile()
if time.time(
) - last_grains_refresh >= __opts__['grains_refresh_frequency']:
log.info('Refreshing grains')
refresh_grains()
last_grains_refresh = time.time()
# Emit syslog at grains refresh frequency
if not (salt.utils.platform.is_windows()) and __opts__.get(
'emit_grains_to_syslog', True):
default_grains_to_emit = [
'system_uuid', 'hubble_uuid', 'session_uuid', 'machine_id',
'splunkindex', 'cloud_details', 'hubble_version',
'localhost', 'fqdn'
]
grains_to_emit = []
grains_to_emit.extend(
__opts__.get('emit_grains_to_syslog_list',
default_grains_to_emit))
emit_to_syslog(grains_to_emit)
try:
log.debug('Executing schedule')
sf_count = schedule()
if sf_count > 0:
log.debug('Executed %d schedule item(s)', sf_count)
hubblestack.log.workaround_salt_log_handler_queues()
except Exception as e:
log.exception('Error executing schedule: {0}'.format(e))
if isinstance(e, KeyboardInterrupt):
raise e
time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))
def refresh_grains(initial=False):
"""
Refresh the grains, pillar, utils, modules, and returners
"""
global __opts__
global __grains__
global __utils__
global __mods__
global __pillar__
global __returners__
global __context__
# 'POP' is for tracking persistent opts protection
if os.environ.get("NOISY_POP_DEBUG"):
log.error("POP refreshing grains (id=%d)", id(__opts__))
persist, old_grains = {}, {}
if initial:
if not os.environ.get("NO_PRESERVE_OPTS"):
if os.environ.get("NOISY_POP_DEBUG"):
log.error("POP setting __opts__ to preservable (id=%d)",
id(__opts__))
hubblestack.loader.set_preservable_opts(__opts__)
elif os.environ.get("NOISY_POP_DEBUG"):
log.error(
"POP we are not attemting to protect __opts__ from lazyloader reloads"
)
else:
old_grains = copy.deepcopy(__grains__)
for grain in __opts__.get("grains_persist", []):
if grain in __grains__:
persist[grain] = __grains__[grain]
# Hardcode these core grains as persisting
persist = {
grain: __grains__[grain]
for grain in ["hubble_version", "buildinfo"] if grain in __grains__
}
if initial:
__context__ = {}
if "grains" in __opts__:
__opts__.pop("grains")
if "pillar" in __opts__:
__opts__.pop("pillar")
__grains__ = hubblestack.loader.grains(__opts__)
__grains__.update(persist)
__grains__["session_uuid"] = SESSION_UUID
# This was a weird one. In older versions of hubble the version and
# buildinfo were not persisted automatically which means that if you
# installed a new version without restarting hubble, grains refresh could
# cause that old daemon to report grains as if it were the new version.
# Now if this hubble_marker_3 grain is present you know you can trust the
# hubble_version and buildinfo.
__grains__["hubble_marker_3"] = True
old_grains.update(__grains__)
__grains__ = old_grains
# Check for default gateway and fall back if necessary
if __grains__.get(
"ip_gw",
None) is False and "fallback_fileserver_backend" in __opts__:
log.info(
"No default gateway detected; using fallback_fileserver_backend.")
__opts__["fileserver_backend"] = __opts__[
"fallback_fileserver_backend"]
__opts__["hubble_uuid"] = __grains__.get("hubble_uuid", None)
__opts__["system_uuid"] = __grains__.get("system_uuid", None)
__pillar__ = {}
__opts__["grains"] = __grains__
__opts__["pillar"] = __pillar__
__utils__ = hubblestack.loader.utils(__opts__)
__mods__ = hubblestack.loader.modules(__opts__,
utils=__utils__,
context=__context__)
__returners__ = hubblestack.loader.returners(__opts__, __mods__)
# the only things that turn up in here (and that get preserved)
# are pulsar.queue, pulsar.notifier and cp.fileclient_###########
# log.debug('keys in __context__: {}'.format(list(__context__)))
hubblestack.utils.stdrec.__grains__ = __grains__
hubblestack.utils.stdrec.__opts__ = __opts__
hubblestack.hec.opt.__grains__ = __grains__
hubblestack.hec.opt.__mods__ = __mods__
hubblestack.hec.opt.__opts__ = __opts__
hubblestack.log.splunk.__grains__ = __grains__
hubblestack.log.splunk.__mods__ = __mods__
hubblestack.log.splunk.__opts__ = __opts__
hubblestack.status.__opts__ = __opts__
hubblestack.status.__mods__ = __mods__
hubblestack.utils.signing.__opts__ = __opts__
hubblestack.utils.signing.__mods__ = __mods__
hubblestack.module_runner.runner.__mods__ = __mods__
hubblestack.module_runner.runner.__grains__ = __grains__
hubblestack.module_runner.runner.__opts__ = __opts__
hubblestack.module_runner.audit_runner.__mods__ = __mods__
hubblestack.module_runner.audit_runner.__grains__ = __grains__
hubblestack.module_runner.audit_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__mods__ = __mods__
hubblestack.module_runner.fdg_runner.__grains__ = __grains__
hubblestack.module_runner.fdg_runner.__opts__ = __opts__
hubblestack.module_runner.fdg_runner.__returners__ = __returners__
hubblestack.utils.signing.__mods__ = __mods__
HSS.start_sigusr1_signal_handler()
hubblestack.log.refresh_handler_std_info()
clear_selective_context()
if not initial and __mods__["config.get"]("splunklogging", False):
hubblestack.log.emit_to_splunk(__grains__, "INFO",
"hubblestack.grains_report")
