def _setup_cached_uuid(): """ Get the cached uuid and cached system uui path, read the files and remove the cached uuid """ # Check for a cloned system with existing hubble_uuid def _get_uuid_from_system(): query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv' # Prefer our /opt/osquery/osqueryi if present osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi', '/usr/bin/osqueryi') for path in osqueryipaths: if hubblestack.utils.path.which(path): live_uuid = hubblestack.modules.cmdmod.run_stdout( '{0} {1}'.format(path, query), output_loglevel='quiet') live_uuid = str(live_uuid).upper() if len(live_uuid) == 36: return live_uuid return None # If osquery isn't available, attempt to get uuid from /sys path (linux only) try: with open('/sys/devices/virtual/dmi/id/product_uuid', 'r') as product_uuid_file: file_uuid = product_uuid_file.read() file_uuid = str(file_uuid).upper() if len(file_uuid) == 36: return file_uuid return None except Exception: return None cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']), 'hubble_cached_uuid') cached_system_uuid_path = os.path.join( os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid') try: if os.path.isfile(cached_uuid_path) and os.path.isfile( cached_system_uuid_path): with open(cached_uuid_path, 'r') as cached_uuid_file, \ open(cached_system_uuid_path, 'r') as cached_system_uuid_file: cached_uuid = cached_uuid_file.read() cached_system_uuid = cached_system_uuid_file.read() if cached_uuid != cached_system_uuid: live_uuid = _get_uuid_from_system() if live_uuid != cached_system_uuid: log.error( "potentially cloned system detected: System_uuid grain " "previously saved on disk doesn't match live system value.\n" "Resettig cached hubble_uuid value.") os.remove(cached_uuid_path) except Exception: log.exception( "Problem opening cache files while checking for previously cloned system" )
def run_function(): ''' Run a single function requested by the user ''' # Parse the args args = [] kwargs = {} for arg in __opts__['args']: if '=' in arg: kwarg, _, value = arg.partition('=') kwargs[kwarg] = value else: args.append(arg) log.debug('Parsed args: {0} | Parsed kwargs: {1}'.format(args, kwargs)) log.info('Executing user-requested function {0}'.format( __opts__['function'])) try: ret = __salt__[__opts__['function']](*args, **kwargs) except KeyError: log.error('Function {0} is not available, or not valid.'.format( __opts__['function'])) sys.exit(1) if __opts__['return']: returner = '{0}.returner'.format(__opts__['return']) if returner not in __returners__: log.error('Could not find {0} returner.'.format(returner)) else: log.info('Returning job data to {0}'.format(returner)) returner_ret = { 'id': __grains__['id'], 'jid': salt.utils.jid.gen_jid(__opts__), 'fun': __opts__['function'], 'fun_args': args + ([kwargs] if kwargs else []), 'return': ret, 'retry': False } if __opts__.get('returner_retry', False): returner_ret['retry'] = True __returners__[returner](returner_ret) # TODO instantiate the salt outputter system? if __opts__['json_print']: print(json.dumps(ret)) else: if not __opts__['no_pprint']: pprint.pprint(ret) else: print(ret)
def run_function(): """ Run a single function requested by the user """ # Parse the args args = [] kwargs = {} for arg in __opts__['args']: if '=' in arg: kwarg, _, value = arg.partition('=') kwargs[kwarg] = value else: args.append(arg) log.debug('Parsed args: %s | Parsed kwargs: %s', args, kwargs) log.info('Executing user-requested function %s', __opts__['function']) mod_fun = __mods__.get(__opts__['function']) if not mod_fun or not callable(mod_fun): log.error('Function %s is not available, or not valid.', __opts__['function']) sys.exit(1) ret = mod_fun(*args, **kwargs) if __opts__['return']: returner = '{0}.returner'.format(__opts__['return']) if returner not in __returners__: log.error('Could not find %s returner.', returner) else: log.info('Returning job data to %s', returner) returner_ret = { 'id': __grains__['id'], 'jid': hubblestack.utils.jid.gen_jid(__opts__), 'fun': __opts__['function'], 'fun_args': args + ([kwargs] if kwargs else []), 'return': ret } __returners__[returner](returner_ret) # TODO instantiate the salt outputter system? if __opts__['json_print']: print(json.dumps(ret)) else: if not __opts__['no_pprint']: pprint.pprint(ret) else: print(ret)
def run_function(): """ Run a single function requested by the user """ # Parse the args args = [] kwargs = {} for arg in __opts__["args"]: if "=" in arg: kwarg, _, value = arg.partition("=") kwargs[kwarg] = value else: args.append(arg) log.debug("Parsed args: %s | Parsed kwargs: %s", args, kwargs) log.info("Executing user-requested function %s", __opts__["function"]) mod_fun = __mods__.get(__opts__["function"]) if not mod_fun or not callable(mod_fun): log.error("Function %s is not available, or not valid.", __opts__["function"]) sys.exit(1) ret = mod_fun(*args, **kwargs) if __opts__["return"]: returner = "{0}.returner".format(__opts__["return"]) if returner not in __returners__: log.error("Could not find %s returner.", returner) else: log.info("Returning job data to %s", returner) returner_ret = { "id": __grains__["id"], "jid": hubblestack.utils.jid.gen_jid(__opts__), "fun": __opts__["function"], "fun_args": args + ([kwargs] if kwargs else []), "return": ret, } __returners__[returner](returner_ret) # TODO instantiate the salt outputter system? if __opts__["json_print"]: print(json.dumps(ret)) else: if not __opts__["no_pprint"]: pprint.pprint(ret) else: print(ret)
def publish(report_directly_to_splunk=True, remove_dots=True, *args): """ Publishes config to splunk at an interval defined in schedule report_directly_to_splunk Whether to emit directly to splunk in addition to returning as a normal job. Defaults to True. remove_dots Whether to replace dots in top-level keys with underscores for ease of handling in splunk. Defaults to True. *args Tuple of opts to log (keys in __opts__). Only those key-value pairs would be published, keys for which are in *args If not passed, entire __opts__ (excluding password/token) would be published """ log.debug("Started publishing config to splunk") opts_to_log = {} if not args: opts_to_log = copy.deepcopy(__opts__) if "grains" in opts_to_log: opts_to_log.pop("grains") else: for arg in args: if arg in __opts__: opts_to_log[arg] = __opts__[arg] # 'POP' is for tracking persistent opts protection if os.environ.get("NOISY_POP_DEBUG"): log.error("POP config_publish (id=%d)", id(__opts__)) filtered_conf = hubblestack.log.filter_logs(opts_to_log, remove_dots=remove_dots) if report_directly_to_splunk: hubblestack.log.emit_to_splunk(filtered_conf, "INFO", "hubblestack.hubble_config") log.debug("Published config to splunk") return filtered_conf
def _execute_function(jobdata, func, returners, args, kwargs): """ Run the scheduled function """ log.debug('Executing scheduled function %s', func) jobdata['last_run'] = time.time() ret = __salt__[func](*args, **kwargs) if __opts__['log_level'] == 'debug': log.debug('Job returned:\n%s', ret) for returner in returners: returner = '{0}.returner'.format(returner) if returner not in __returners__: log.error('Could not find %s returner.', returner) continue log.debug('Returning job data to %s', returner) returner_ret = {'id': __grains__['id'], 'jid': salt.utils.jid.gen_jid(__opts__), 'fun': func, 'fun_args': args + ([kwargs] if kwargs else []), 'return': ret} __returners__[returner](returner_ret)
def _execute_function(jobdata, func, returners, args, kwargs): """Run the scheduled function""" log.debug("Executing scheduled function %s", func) jobdata["last_run"] = time.time() ret = __mods__[func](*args, **kwargs) if __opts__["log_level"] == "debug": log.debug("Job returned:\n%s", ret) for returner in returners: returner = "{0}.returner".format(returner) if returner not in __returners__: log.error("Could not find %s returner.", returner) continue log.debug("Returning job data to %s", returner) returner_ret = { "id": __grains__["id"], "jid": hubblestack.utils.jid.gen_jid(__opts__), "fun": func, "fun_args": args + ([kwargs] if kwargs else []), "return": ret, } __returners__[returner](returner_ret)
def schedule(): """ Rudimentary single-pass scheduler If we find we miss some of the salt scheduler features we could potentially pull in some of that code. Schedule data should be placed in the config with the following format: .. code-block:: yaml schedule: job1: function: hubble.audit seconds: 3600 splay: 100 min_splay: 50 args: - cis.centos-7-level-1-scored-v2-1-0 kwargs: verbose: True show_profile: True returner: splunk_nova_return run_on_start: True Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a scheduled job must always have a ``function`` and a time in ``seconds`` of how often to run the job. function Function to run in the format ``<module>.<function>``. Technically any salt module can be run in this way, but we recommend sticking to hubble functions. For simplicity, functions are run in the main daemon thread, so overloading the scheduler can result in functions not being run in a timely manner. seconds Frequency with which the job should be run, in seconds splay Randomized splay for the job, in seconds. A random number between <min_splay> and <splay> will be chosen and added to the ``seconds`` argument, to decide the true frequency. The splay will be chosen on first run, and will only change when the daemon is restarted. Optional. min_splay This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it defaults to zero. Optional. args List of arguments for the function. Optional. kwargs Dict of keyword arguments for the function. Optional. returner String with a single returner, or list of returners to which we should send the results. Optional. run_on_start Whether to run the scheduled job on daemon start. Defaults to False. Optional. """ sf_count = 0 base = datetime(2018, 1, 1, 0, 0) schedule_config = __opts__.get('schedule', {}) if 'user_schedule' in __opts__ and isinstance(__opts__['user_schedule'], dict): schedule_config.update(__opts__['user_schedule']) for jobname, jobdata in schedule_config.items(): try: # Error handling galore if not jobdata or not isinstance(jobdata, dict): log.error('Scheduled job %s does not have valid data', jobname) continue if 'function' not in jobdata or 'seconds' not in jobdata: log.error( 'Scheduled job %s is missing a ``function`` or ``seconds`` argument', jobname) continue func = jobdata['function'] if func not in __mods__: log.error( 'Scheduled job %s has a function %s which could not be found.', jobname, func) continue try: if 'cron' in jobdata: seconds = getsecondsbycronexpression(base, jobdata['cron']) else: seconds = int(jobdata['seconds']) splay = int(jobdata.get('splay', 0)) min_splay = int(jobdata.get('min_splay', 0)) except ValueError: log.error( 'Scheduled job %s has an invalid value for seconds or splay.', jobname) args = jobdata.get('args', []) if not isinstance(args, list): log.error('Scheduled job %s has args not formed as a list: %s', jobname, args) kwargs = jobdata.get('kwargs', {}) if not isinstance(kwargs, dict): log.error( 'Scheduled job %s has kwargs not formed as a dict: %s', jobname, kwargs) returners = jobdata.get('returner', []) if not isinstance(returners, list): returners = [returners] # Actually process the job run = _process_job(jobdata, splay, seconds, min_splay, base) if run: _execute_function(jobdata, func, returners, args, kwargs) sf_count += 1 except: log.error( "Exception in running job: %s; continuing with next job...", jobname, exc_info=True) return sf_count
def kill_other_or_sys_exit(xpid, hname=r'hubble', ksig=signal.SIGTERM, kill_other=True, no_pgrp=True): """ Attempt to locate other hubbles using a cmdline regular expression and kill them when found. If killing the other processes fails (or kill_other is False), sys.exit instead. params: hname :- the regular expression pattern to use to locate hubble (default: hubble) ksig :- the signal to use to kill the other processes (default: signal.SIGTERM=15) kill_other :- (default: True); when false, don't attempt to kill, just locate and exit (if found) no_pgrp :- Avoid killing processes in this pgrp (avoid suicide). When no_pgrp is True, invoke os.getprgp() to populate the actual value. caveats: There are some detailed notes on the process scanning in the function as comments. The most important caveat is that the hname regular expressions must match expecting that /proc/$$/cmdline text is null separated, not space separated. The other main caveat is that we can't actually examine the /proc/$$/exe file (that's always just a python). We have to scan the invocation text the kernel stored at launch. That text is not immutable and should not (normally) be relied upon for any purpose -- and this method does rely on it. """ if no_pgrp is True: no_pgrp = os.getpgrp() if isinstance(no_pgrp, int): no_pgrp = str(no_pgrp) if os.path.isdir("/proc/{pid}".format(pid=xpid)): # NOTE: we'd prefer to check readlink(/proc/[pid]/exe), but that won't do # any good the /opt/whatever/bin/hubble is normally a text file with a # shebang; which the kernel picks up and uses to execute the real binary # with the "bin" file as an argument; so we'll have to live with cmdline pfile = '/proc/{pid}/cmdline'.format(pid=xpid) log.debug('searching %s for hubble procs matching %s', pfile, hname) with open(pfile, 'r') as pidfile: # NOTE: cmdline is actually null separated, not space separated # that shouldn't matter much for most hname regular expressions, but one never knows. cmdline = pidfile.readline().replace('\x00', ' ').strip() if re.search(hname, cmdline): if no_pgrp: pstatfile = '/proc/{pid}/stat'.format(pid=xpid) with open(pstatfile, 'r') as fh2: # NOTE: man proc(5) § /proc/[pid]/stat # (pid, comm, state, ppid, pgrp, session, tty_nr, tpgid, flags, ...) pgrp = fh2.readline().split()[4] if pgrp == no_pgrp: log.debug( "process (%s) exists and seems to be a hubble, " "but matches our process group (%s), ignoring", xpid, pgrp) return False if kill_other: log.warn( "process seems to still be alive and seems to be hubble," " attempting to shutdown") os.kill(int(xpid), ksig) time.sleep(1) if os.path.isdir("/proc/{pid}".format(pid=xpid)): log.error( "fatal error: failed to shutdown process (pid=%s) successfully", xpid) sys.exit(1) else: return True else: log.error( "refusing to run while another hubble instance is running") sys.exit(1) else: # pidfile present, but nothing at that pid. Did we receive a sigterm? log.warning( 'Pidfile found on startup, but no process at that pid. Did hubble receive a SIGTERM?' ) return False
def load_config(): ''' Load the config from configfile and load into imported salt modules ''' # Parse arguments parsed_args = parse_args() # Let's find out the path of this module if 'SETUP_DIRNAME' in globals(): # This is from the exec() call in Salt's setup.py this_file = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py') # pylint: disable=E0602 else: this_file = __file__ install_dir = os.path.dirname(os.path.realpath(this_file)) # Load unique data for Windows or Linux if salt.utils.platform.is_windows(): if parsed_args.get('configfile') is None: parsed_args[ 'configfile'] = 'C:\\Program Files (x86)\\Hubble\\etc\\hubble\\hubble.conf' salt.config.DEFAULT_MINION_OPTS[ 'cachedir'] = 'C:\\Program Files (x86)\\hubble\\var\\cache' salt.config.DEFAULT_MINION_OPTS[ 'pidfile'] = 'C:\\Program Files (x86)\\hubble\\var\\run\\hubble.pid' salt.config.DEFAULT_MINION_OPTS[ 'log_file'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble.log' salt.config.DEFAULT_MINION_OPTS[ 'osquery_dbpath'] = 'C:\\Program Files (x86)\\hubble\\var\\hubble_osquery_db' salt.config.DEFAULT_MINION_OPTS[ 'osquerylogpath'] = 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery' salt.config.DEFAULT_MINION_OPTS['osquerylog_backupdir'] = \ 'C:\\Program Files (x86)\\hubble\\var\\log\\hubble_osquery\\backuplogs' else: if parsed_args.get('configfile') is None: parsed_args['configfile'] = '/etc/hubble/hubble' salt.config.DEFAULT_MINION_OPTS['cachedir'] = '/var/cache/hubble' salt.config.DEFAULT_MINION_OPTS['pidfile'] = '/var/run/hubble.pid' salt.config.DEFAULT_MINION_OPTS['log_file'] = '/var/log/hubble' salt.config.DEFAULT_MINION_OPTS[ 'osquery_dbpath'] = '/var/cache/hubble/osquery' salt.config.DEFAULT_MINION_OPTS[ 'osquerylogpath'] = '/var/log/hubble_osquery' salt.config.DEFAULT_MINION_OPTS[ 'osquerylog_backupdir'] = '/var/log/hubble_osquery/backuplogs' salt.config.DEFAULT_MINION_OPTS['file_roots'] = {'base': []} salt.config.DEFAULT_MINION_OPTS['log_level'] = 'error' salt.config.DEFAULT_MINION_OPTS['file_client'] = 'local' salt.config.DEFAULT_MINION_OPTS[ 'fileserver_update_frequency'] = 43200 # 12 hours salt.config.DEFAULT_MINION_OPTS[ 'grains_refresh_frequency'] = 3600 # 1 hour salt.config.DEFAULT_MINION_OPTS['scheduler_sleep_frequency'] = 0.5 salt.config.DEFAULT_MINION_OPTS['default_include'] = 'hubble.d/*.conf' salt.config.DEFAULT_MINION_OPTS['logfile_maxbytes'] = 100000000 # 100MB salt.config.DEFAULT_MINION_OPTS[ 'logfile_backups'] = 1 # maximum rotated logs salt.config.DEFAULT_MINION_OPTS[ 'delete_inaccessible_azure_containers'] = False salt.config.DEFAULT_MINION_OPTS[ 'enable_globbing_in_nebula_masking'] = False # Globbing will not be supported in nebula masking salt.config.DEFAULT_MINION_OPTS[ 'osquery_logfile_maxbytes'] = 50000000 # 50MB salt.config.DEFAULT_MINION_OPTS[ 'osquery_logfile_maxbytes_toparse'] = 100000000 #100MB salt.config.DEFAULT_MINION_OPTS['osquery_backuplogs_count'] = 2 global __opts__ __opts__ = salt.config.minion_config(parsed_args.get('configfile')) __opts__.update(parsed_args) __opts__['conf_file'] = parsed_args.get('configfile') __opts__['install_dir'] = install_dir if __opts__['version']: print(__version__) clean_up_process(None, None) sys.exit(0) if __opts__['buildinfo']: try: from hubblestack import __buildinfo__ except ImportError: __buildinfo__ = 'NOT SET' print(__buildinfo__) clean_up_process(None, None) sys.exit(0) scan_proc = __opts__.get('scan_proc', False) if __opts__['daemonize']: # before becoming a daemon, check for other procs and possibly send # them a signal 15 (otherwise refuse to run) if not __opts__.get('ignore_running', False): check_pidfile(kill_other=True, scan_proc=scan_proc) salt.utils.daemonize() create_pidfile() elif not __opts__['function'] and not __opts__['version'] and not __opts__[ 'buildinfo']: # check the pidfile and possibly refuse to run # (assuming this isn't a single function call) if not __opts__.get('ignore_running', False): check_pidfile(kill_other=False, scan_proc=scan_proc) signal.signal(signal.SIGTERM, clean_up_process) signal.signal(signal.SIGINT, clean_up_process) signal.signal(signal.SIGABRT, clean_up_process) signal.signal(signal.SIGFPE, clean_up_process) signal.signal(signal.SIGILL, clean_up_process) signal.signal(signal.SIGSEGV, clean_up_process) if not salt.utils.platform.is_windows(): signal.signal(signal.SIGHUP, clean_up_process) signal.signal(signal.SIGQUIT, clean_up_process) # Optional sleep to wait for network time.sleep(int(__opts__.get('startup_sleep', 0))) # Convert -vvv to log level if __opts__['log_level'] is None: # Default to 'error' __opts__['log_level'] = 'error' # Default to more verbose if we're daemonizing if __opts__['daemonize']: __opts__['log_level'] = 'info' # Handle the explicit -vvv settings if __opts__['verbose'] == 1: __opts__['log_level'] = 'warning' elif __opts__['verbose'] == 2: __opts__['log_level'] = 'info' elif __opts__['verbose'] >= 3: __opts__['log_level'] = 'debug' # Setup module/grain/returner dirs module_dirs = __opts__.get('module_dirs', []) module_dirs.append( os.path.join(os.path.dirname(__file__), 'extmods', 'modules')) __opts__['module_dirs'] = module_dirs grains_dirs = __opts__.get('grains_dirs', []) grains_dirs.append( os.path.join(os.path.dirname(__file__), 'extmods', 'grains')) __opts__['grains_dirs'] = grains_dirs returner_dirs = __opts__.get('returner_dirs', []) returner_dirs.append( os.path.join(os.path.dirname(__file__), 'extmods', 'returners')) __opts__['returner_dirs'] = returner_dirs fileserver_dirs = __opts__.get('fileserver_dirs', []) fileserver_dirs.append( os.path.join(os.path.dirname(__file__), 'extmods', 'fileserver')) __opts__['fileserver_dirs'] = fileserver_dirs utils_dirs = __opts__.get('utils_dirs', []) utils_dirs.append( os.path.join(os.path.dirname(__file__), 'extmods', 'utils')) __opts__['utils_dirs'] = utils_dirs fdg_dirs = __opts__.get('fdg_dirs', []) fdg_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'fdg')) __opts__['fdg_dirs'] = fdg_dirs __opts__['file_roots']['base'].insert( 0, os.path.join(os.path.dirname(__file__), 'files')) if 'roots' not in __opts__['fileserver_backend']: __opts__['fileserver_backend'].append('roots') # Disable all of salt's boto modules, they give nothing but trouble to the loader disable_modules = __opts__.get('disable_modules', []) disable_modules.extend([ 'boto3_elasticache', 'boto3_route53', 'boto3_sns', 'boto_apigateway', 'boto_asg', 'boto_cfn', 'boto_cloudfront', 'boto_cloudtrail', 'boto_cloudwatch_event', 'boto_cloudwatch', 'boto_cognitoidentity', 'boto_datapipeline', 'boto_dynamodb', 'boto_ec2', 'boto_efs', 'boto_elasticache', 'boto_elasticsearch_domain', 'boto_elb', 'boto_elbv2', 'boto_iam', 'boto_iot', 'boto_kinesis', 'boto_kms', 'boto_lambda', 'boto_rds', 'boto_route53', 'boto_s3_bucket', 'boto_s3', 'boto_secgroup', 'boto_sns', 'boto_sqs', 'boto_ssm', 'boto_vpc', ]) __opts__['disable_modules'] = disable_modules # Console logging is probably the same, but can be different console_logging_opts = { 'log_level': __opts__.get('console_log_level', __opts__['log_level']), 'log_format': __opts__.get('console_log_format', '%(asctime)s [%(levelname)-5s] %(message)s'), 'date_format': __opts__.get('console_log_date_format', '%H:%M:%S'), } file_logging_opts = { 'log_file': __opts__.get('log_file', '/var/log/hubble'), 'log_level': __opts__['log_level'], 'log_format': __opts__.get( 'log_format', '%(asctime)s,%(msecs)03d [%(levelname)-5s] [%(name)s:%(lineno)d] %(message)s' ), 'date_format': __opts__.get('log_date_format', '%Y-%m-%d %H:%M:%S'), 'max_bytes': __opts__.get('logfile_maxbytes', 100000000), 'backup_count': __opts__.get('logfile_backups', 1), } # Setup logging hubblestack.log.setup_console_logger(**console_logging_opts) hubblestack.log.setup_file_logger(**file_logging_opts) # 384 is 0o600 permissions, written without octal for python 2/3 compat os.chmod(__opts__['log_file'], 384) os.chmod(parsed_args.get('configfile'), 384) # Check for a cloned system with existing hubble_uuid def _get_uuid_from_system(): query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv' # Prefer our /opt/osquery/osqueryi if present osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi', '/usr/bin/osqueryi') for path in osqueryipaths: if salt.utils.path.which(path): live_uuid = salt.modules.cmdmod.run_stdout( '{0} {1}'.format(path, query), output_loglevel='quiet') live_uuid = str(live_uuid).upper() if len(live_uuid) == 36: return live_uuid else: return None # If osquery isn't available, attempt to get uuid from /sys path (linux only) try: with open('/sys/devices/virtual/dmi/id/product_uuid', 'r') as f: file_uuid = f.read() file_uuid = str(file_uuid).upper() if len(file_uuid) == 36: return file_uuid else: return None except Exception: return None cached_uuid_path = os.path.join(os.path.dirname(__opts__['configfile']), 'hubble_cached_uuid') cached_system_uuid_path = os.path.join( os.path.dirname(__opts__['configfile']), 'hubble_cached_system_uuid') try: if os.path.isfile(cached_uuid_path) and os.path.isfile( cached_system_uuid_path): with open(cached_uuid_path, 'r') as f, open(cached_system_uuid_path, 'r') as g: cached_uuid = f.read() cached_system_uuid = g.read() if cached_uuid != cached_system_uuid: live_uuid = _get_uuid_from_system() if live_uuid != cached_system_uuid: log.error( "potentially cloned system detected: System_uuid grain " "previously saved on disk doesn't match live system value.\n" "Resettig cached hubble_uuid value.") os.remove(cached_uuid_path) except Exception: log.exception( "Problem opening cache files while checking for previously cloned system" ) refresh_grains(initial=True) if __salt__['config.get']('splunklogging', False): hubblestack.log.setup_splunk_logger() hubblestack.log.emit_to_splunk(__grains__, 'INFO', 'hubblestack.grains_report', remove_sensitive_logs=True)
def schedule(): ''' Rudimentary single-pass scheduler If we find we miss some of the salt scheduler features we could potentially pull in some of that code. Schedule data should be placed in the config with the following format: .. code-block:: yaml schedule: job1: function: hubble.audit seconds: 3600 splay: 100 min_splay: 50 args: - cis.centos-7-level-1-scored-v2-1-0 kwargs: verbose: True show_profile: True returner: splunk_nova_return run_on_start: True Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a scheduled job must always have a ``function`` and a time in ``seconds`` of how often to run the job. function Function to run in the format ``<module>.<function>``. Technically any salt module can be run in this way, but we recommend sticking to hubble functions. For simplicity, functions are run in the main daemon thread, so overloading the scheduler can result in functions not being run in a timely manner. seconds Frequency with which the job should be run, in seconds splay Randomized splay for the job, in seconds. A random number between <min_splay> and <splay> will be chosen and added to the ``seconds`` argument, to decide the true frequency. The splay will be chosen on first run, and will only change when the daemon is restarted. Optional. min_splay This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it defaults to zero. Optional. args List of arguments for the function. Optional. kwargs Dict of keyword arguments for the function. Optional. returner String with a single returner, or list of returners to which we should send the results. Optional. run_on_start Whether to run the scheduled job on daemon start. Defaults to False. Optional. ''' sf_count = 0 base = datetime(2018, 1, 1, 0, 0) schedule_config = __opts__.get('schedule', {}) if 'user_schedule' in __opts__ and isinstance(__opts__['user_schedule'], dict): schedule_config.update(__opts__['user_schedule']) for jobname, jobdata in schedule_config.iteritems(): # Error handling galore if not jobdata or not isinstance(jobdata, dict): log.error( 'Scheduled job {0} does not have valid data'.format(jobname)) continue if 'function' not in jobdata or 'seconds' not in jobdata: log.error('Scheduled job {0} is missing a ``function`` or ' '``seconds`` argument'.format(jobname)) continue func = jobdata['function'] if func not in __salt__: log.error('Scheduled job {0} has a function {1} which could not ' 'be found.'.format(jobname, func)) continue try: if 'cron' in jobdata: seconds = getsecondsbycronexpression(base, jobdata['cron']) else: seconds = int(jobdata['seconds']) splay = int(jobdata.get('splay', 0)) min_splay = int(jobdata.get('min_splay', 0)) except ValueError: log.error('Scheduled job {0} has an invalid value for seconds or ' 'splay.'.format(jobname)) args = jobdata.get('args', []) if not isinstance(args, list): log.error( 'Scheduled job {0} has args not formed as a list: {1}'.format( jobname, args)) kwargs = jobdata.get('kwargs', {}) if not isinstance(kwargs, dict): log.error('Scheduled job {0} has kwargs not formed as a dict: {1}'. format(jobname, kwargs)) returners = jobdata.get('returner', []) if not isinstance(returners, list): returners = [returners] returner_retry = jobdata.get('returner_retry', False) # Actually process the job run = False if 'last_run' not in jobdata: if jobdata.get('run_on_start', False): if splay: # Run `splay` seconds in the future, by telling the scheduler we last ran it # `seconds - splay` seconds ago. jobdata['last_run'] = time.time() - ( seconds - random.randint(min_splay, splay)) else: # Run now run = True jobdata['last_run'] = time.time() else: if splay: # Run `seconds + splay` seconds in the future by telling the scheduler we last # ran it at now + `splay` seconds. jobdata['last_run'] = time.time() + random.randint( min_splay, splay) elif 'buckets' in jobdata: # Place the host in a bucket and fix the execution time. jobdata['last_run'] = getlastrunbybuckets( jobdata['buckets'], seconds) log.debug('last_run according to bucket is {0}'.format( jobdata['last_run'])) elif 'cron' in jobdata: # execute the hubble process based on cron expression jobdata['last_run'] = getlastrunbycron(base, seconds) else: # Run in `seconds` seconds. jobdata['last_run'] = time.time() if jobdata['last_run'] < time.time() - seconds: run = True if run: log.debug('Executing scheduled function {0}'.format(func)) jobdata['last_run'] = time.time() ret = __salt__[func](*args, **kwargs) sf_count += 1 if __opts__['log_level'] == 'debug': log.debug('Job returned:\n{0}'.format(ret)) for returner in returners: returner = '{0}.returner'.format(returner) if returner not in __returners__: log.error('Could not find {0} returner.'.format(returner)) continue log.debug('Returning job data to {0}'.format(returner)) returner_ret = { 'id': __grains__['id'], 'jid': salt.utils.jid.gen_jid(__opts__), 'fun': func, 'fun_args': args + ([kwargs] if kwargs else []), 'return': ret, 'retry': returner_retry } __returners__[returner](returner_ret) return sf_count
def refresh_grains(initial=False): """ Refresh the grains, pillar, utils, modules, and returners """ global __opts__ global __grains__ global __utils__ global __mods__ global __pillar__ global __returners__ global __context__ # 'POP' is for tracking persistent opts protection if os.environ.get("NOISY_POP_DEBUG"): log.error("POP refreshing grains (id=%d)", id(__opts__)) persist, old_grains = {}, {} if initial: if not os.environ.get("NO_PRESERVE_OPTS"): if os.environ.get("NOISY_POP_DEBUG"): log.error("POP setting __opts__ to preservable (id=%d)", id(__opts__)) hubblestack.loader.set_preservable_opts(__opts__) elif os.environ.get("NOISY_POP_DEBUG"): log.error( "POP we are not attemting to protect __opts__ from lazyloader reloads" ) else: old_grains = copy.deepcopy(__grains__) for grain in __opts__.get("grains_persist", []): if grain in __grains__: persist[grain] = __grains__[grain] # Hardcode these core grains as persisting persist = { grain: __grains__[grain] for grain in ["hubble_version", "buildinfo"] if grain in __grains__ } if initial: __context__ = {} if "grains" in __opts__: __opts__.pop("grains") if "pillar" in __opts__: __opts__.pop("pillar") __grains__ = hubblestack.loader.grains(__opts__) __grains__.update(persist) __grains__["session_uuid"] = SESSION_UUID # This was a weird one. In older versions of hubble the version and # buildinfo were not persisted automatically which means that if you # installed a new version without restarting hubble, grains refresh could # cause that old daemon to report grains as if it were the new version. # Now if this hubble_marker_3 grain is present you know you can trust the # hubble_version and buildinfo. __grains__["hubble_marker_3"] = True old_grains.update(__grains__) __grains__ = old_grains # Check for default gateway and fall back if necessary if __grains__.get( "ip_gw", None) is False and "fallback_fileserver_backend" in __opts__: log.info( "No default gateway detected; using fallback_fileserver_backend.") __opts__["fileserver_backend"] = __opts__[ "fallback_fileserver_backend"] __opts__["hubble_uuid"] = __grains__.get("hubble_uuid", None) __opts__["system_uuid"] = __grains__.get("system_uuid", None) __pillar__ = {} __opts__["grains"] = __grains__ __opts__["pillar"] = __pillar__ __utils__ = hubblestack.loader.utils(__opts__) __mods__ = hubblestack.loader.modules(__opts__, utils=__utils__, context=__context__) __returners__ = hubblestack.loader.returners(__opts__, __mods__) # the only things that turn up in here (and that get preserved) # are pulsar.queue, pulsar.notifier and cp.fileclient_########### # log.debug('keys in __context__: {}'.format(list(__context__))) hubblestack.utils.stdrec.__grains__ = __grains__ hubblestack.utils.stdrec.__opts__ = __opts__ hubblestack.hec.opt.__grains__ = __grains__ hubblestack.hec.opt.__mods__ = __mods__ hubblestack.hec.opt.__opts__ = __opts__ hubblestack.log.splunk.__grains__ = __grains__ hubblestack.log.splunk.__mods__ = __mods__ hubblestack.log.splunk.__opts__ = __opts__ hubblestack.status.__opts__ = __opts__ hubblestack.status.__mods__ = __mods__ hubblestack.utils.signing.__opts__ = __opts__ hubblestack.utils.signing.__mods__ = __mods__ hubblestack.module_runner.runner.__mods__ = __mods__ hubblestack.module_runner.runner.__grains__ = __grains__ hubblestack.module_runner.runner.__opts__ = __opts__ hubblestack.module_runner.audit_runner.__mods__ = __mods__ hubblestack.module_runner.audit_runner.__grains__ = __grains__ hubblestack.module_runner.audit_runner.__opts__ = __opts__ hubblestack.module_runner.fdg_runner.__mods__ = __mods__ hubblestack.module_runner.fdg_runner.__grains__ = __grains__ hubblestack.module_runner.fdg_runner.__opts__ = __opts__ hubblestack.module_runner.fdg_runner.__returners__ = __returners__ hubblestack.utils.signing.__mods__ = __mods__ HSS.start_sigusr1_signal_handler() hubblestack.log.refresh_handler_std_info() clear_selective_context() if not initial and __mods__["config.get"]("splunklogging", False): hubblestack.log.emit_to_splunk(__grains__, "INFO", "hubblestack.grains_report")