def main(): hosts = None action = None path = None status = None dryrun = False store_filepath = "/etc/data_retention/dataretention_rules.sq3" try: (options, remainder) = getopt.gnu_getopt( sys.argv[1:], "H:a:p:s:r:dh", ["hosts=", "action=", "path=", "status=", "dryrun", "help"]) except getopt.GetoptError as err: usage("Unknown option specified: " + str(err)) for (opt, val) in options: if opt in ["-H", "--hosts"]: hosts = val elif opt in ["-a", "--action"]: action = val elif opt in ["-p", "--path"]: path = val elif opt in ["-s", "--status"]: status = val elif opt in ["-r", "--rulestore"]: store_filepath = val elif opt in ["-d", "--dryrun"]: dryrun = True elif opt in ["-h", "--help"]: usage() else: usage("Unknown option specified: %s" % opt) if len(remainder) > 0: usage("Unknown option specified: <%s>" % remainder[0]) check_args(hosts, action, status) if not os.path.exists(store_filepath): usage('no such rulestore at %s' % store_filepath) cdb = RuleStore(store_filepath) cdb.store_db_init(None) hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts) # if we are given one host, check that the host has a table or whine if htype == 'glob' and '*' not in hosts: if not clouseau.retention.utils.ruleutils.check_host_table_exists( cdb, hosts): usage('no such host in rule store, %s' % hosts) if htype == 'grain' or htype == 'glob': client = LocalClientPlus() hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype) do_action(cdb, action, hosts, status, path, dryrun)
def main(): hosts = None action = None path = None status = None dryrun = False store_filepath = "/etc/data_retention/dataretention_rules.sq3" try: (options, remainder) = getopt.gnu_getopt( sys.argv[1:], "H:a:p:s:r:dh", ["hosts=", "action=", "path=", "status=", "dryrun", "help"]) except getopt.GetoptError as err: usage("Unknown option specified: " + str(err)) for (opt, val) in options: if opt in ["-H", "--hosts"]: hosts = val elif opt in ["-a", "--action"]: action = val elif opt in ["-p", "--path"]: path = val elif opt in ["-s", "--status"]: status = val elif opt in ["-r", "--rulestore"]: store_filepath = val elif opt in ["-d", "--dryrun"]: dryrun = True elif opt in ["-h", "--help"]: usage() else: usage("Unknown option specified: %s" % opt) if len(remainder) > 0: usage("Unknown option specified: <%s>" % remainder[0]) check_args(hosts, action, status) if not os.path.exists(store_filepath): usage('no such rulestore at %s' % store_filepath) cdb = RuleStore(store_filepath) cdb.store_db_init(None) hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts) # if we are given one host, check that the host has a table or whine if htype == 'glob' and '*' not in hosts: if not clouseau.retention.utils.ruleutils.check_host_table_exists(cdb, hosts): usage('no such host in rule store, %s' % hosts) if htype == 'grain' or htype == 'glob': client = LocalClientPlus() hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype) do_action(cdb, action, hosts, status, path, dryrun)
def run_remotely(self): ''' run the current script on specified remote hosts ''' client = LocalClientPlus() if self.expanded_hosts is None: self.expanded_hosts = client.cmd_expandminions( self.hosts, "test.ping", expr_form=self.hosts_expr_type) # fixme instead of this we call the right salt module based on the # audit type and with the self.auditmodule_args which is a list hostbatches = [ self. expanded_hosts[i:i + clouseau.retention.utils.config.conf['batchsize']] for i in range(0, len(self.expanded_hosts), clouseau.retention.utils.config.conf['batchsize']) ] result = {} for hosts in hostbatches: if self.verbose: sys.stderr.write("INFO: running on hosts\n") sys.stderr.write(','.join(hosts) + '\n') path = os.path.join(os.path.dirname(self.store_filepath), "data_retention.d") contents = clouseau.retention.utils.ignores.prep_good_rules_tosend( path, hosts) if contents: new_result = client.cmd_full_return( hosts, 'cp.recv', [contents, os.path.join(self.confdir, 'fromstore')], expr_form='list') # step two: run the appropriate salt audit module function new_result = client.cmd(hosts, "retentionaudit.%s" % self.get_auditfunction_name(), self.auditmodule_args, expr_form='list', timeout=self.timeout) if new_result is not None: result.update(new_result) # fixme, collect and report on hosts that did # not respond return result
def get_salt_known_hosts(self): ''' get all known salt minions by doing a test.ping and getting the list of expanded minions (not of responsive hosts, of known hosts) FIXME there are functions provided for this in more recent versions of salt, use them ''' if self.known_hosts is None: client = LocalClientPlus() # fixme we should just look up the salt keys, cheaper self.known_hosts = client.cmd_expandminions('*', "test.ping", expr_form='glob') return self.known_hosts
def run_remotely(self): ''' run the current script on specified remote hosts ''' client = LocalClientPlus() if self.expanded_hosts is None: self.expanded_hosts = client.cmd_expandminions( self.hosts, "test.ping", expr_form=self.hosts_expr_type) # fixme instead of this we call the right salt module based on the # audit type and with the self.auditmodule_args which is a list hostbatches = [self.expanded_hosts[i: i + clouseau.retention.utils.config.conf['batchsize']] for i in range(0, len(self.expanded_hosts), clouseau.retention.utils.config.conf['batchsize'])] result = {} for hosts in hostbatches: if self.verbose: sys.stderr.write("INFO: running on hosts\n") sys.stderr.write(','.join(hosts) + '\n') path = os.path.join(os.path.dirname(self.store_filepath), "data_retention.d") contents = clouseau.retention.utils.ignores.prep_good_rules_tosend(path, hosts) if contents: new_result = client.cmd_full_return( hosts, 'cp.recv', [contents, os.path.join(self.confdir, 'fromstore')], expr_form='list') # step two: run the appropriate salt audit module function new_result = client.cmd( hosts, "retentionaudit.%s" % self.get_auditfunction_name(), self.auditmodule_args, expr_form='list', timeout=self.timeout) if new_result is not None: result.update(new_result) # fixme, collect and report on hosts that did # not respond return result
def __init__(self, hosts_expr, audit_type, confdir=None, prettyprint=False, show_content=False, dirsizes=False, summary_report=False, depth=2, to_check=None, ignore_also=None, timeout=60, maxfiles=None, store_filepath=None, verbose=False): ''' hosts_expr: list or grain-based or wildcard expr for hosts to be audited audit_type: type of audit e.g. 'logs', 'homes' confdir: directory where the yaml config files are stored prettyprint: nicely format the output display show_content: show the first line or so from problematic files dirsizes: show only directories which have too many files to audit properly, don't report on files at all summary_report: do a summary of results instead of detailed this means different thiings depending on the audit type depth: the auditor will give up if a directory has too any files it (saves it form dying on someone's 25gb homedir). this option tells it how far down the tree to go from the top dir of the audit, before starting to count. e.g. do we count in /home/ariel or separately in /home/ariel/* or in /home/ariel/*/*, etc. to_check: comma-separated list of dirs (must end in '/') and/or files that will be checked; if this is None then all dirs/files will be checked ignore_also: comma-separated list of dirs (must end in '/') and/or files that will be skipped in addition to the ones in the config, rules, etc. timeout: salt timeout for running remote commands maxfiles: how many files in a directory tree is too many to audit (at which point we warn about that and move on) store_filepath: full path to rule store (sqlite3 db) verbose: show informative messages during processing ''' self.hosts_expr = hosts_expr self.audit_type = audit_type self.confdir = confdir self.locations = audit_type + "_locations" self.prettyprint = prettyprint self.show_sample_content = show_content self.dirsizes = dirsizes self.show_summary = summary_report self.depth = depth + 1 # actually count of path separators in dirname self.to_check = to_check self.ignore_also = ignore_also self.timeout = timeout self.store_filepath = store_filepath self.verbose = verbose self.max_files = maxfiles self.set_up_max_files(maxfiles) self.magic = clouseau.retention.utils.magic.magic_open( clouseau.retention.utils.magic.MAGIC_NONE) self.magic.load() self.summary = None self.display_from_dict = FileInfo.display_from_dict self.runner = None if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost": # run locally self.localaudit = True self.expanded_hosts = [] self.cdb = None else: self.localaudit = False clouseau.retention.utils.config.set_up_conf(confdir) client = LocalClientPlus() hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr) self.expanded_hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=expr_type) self.cdb = RuleStore(self.store_filepath) self.cdb.store_db_init(self.expanded_hosts) self.set_up_and_export_rule_store()
def __init__(self, hosts_expr, audit_type, confdir=None, prettyprint=False, show_content=False, dirsizes=False, summary_report=False, depth=2, to_check=None, ignore_also=None, timeout=60, maxfiles=None, store_filepath=None, verbose=False): ''' hosts_expr: list or grain-based or wildcard expr for hosts to be audited audit_type: type of audit e.g. 'logs', 'homes' confdir: directory where the yaml config files are stored prettyprint: nicely format the output display show_content: show the first line or so from problematic files dirsizes: show only directories which have too many files to audit properly, don't report on files at all summary_report: do a summary of results instead of detailed this means different thiings depending on the audit type depth: the auditor will give up if a directory has too any files it (saves it form dying on someone's 25gb homedir). this option tells it how far down the tree to go from the top dir of the audit, before starting to count. e.g. do we count in /home/ariel or separately in /home/ariel/* or in /home/ariel/*/*, etc. to_check: comma-separated list of dirs (must end in '/') and/or files that will be checked; if this is None then all dirs/files will be checked ignore_also: comma-separated list of dirs (must end in '/') and/or files that will be skipped in addition to the ones in the config, rules, etc. timeout: salt timeout for running remote commands maxfiles: how many files in a directory tree is too many to audit (at which point we warn about that and move on) store_filepath: full path to rule store (sqlite3 db) verbose: show informative messages during processing ''' self.hosts_expr = hosts_expr self.audit_type = audit_type self.confdir = confdir self.locations = audit_type + "_locations" self.prettyprint = prettyprint self.show_sample_content = show_content self.dirsizes = dirsizes self.show_summary = summary_report self.depth = depth + 1 # actually count of path separators in dirname self.to_check = to_check self.ignore_also = ignore_also self.timeout = timeout self.store_filepath = store_filepath self.verbose = verbose self.max_files = maxfiles self.set_up_max_files(maxfiles) self.magic = clouseau.retention.utils.magic.magic_open( clouseau.retention.utils.magic.MAGIC_NONE) self.magic.load() self.summary = None self.display_from_dict = FileInfo.display_from_dict self.runner = None if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost": # run locally self.localaudit = True self.expanded_hosts = [] self.cdb = None else: self.localaudit = False clouseau.retention.utils.config.set_up_conf(confdir) client = LocalClientPlus() hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr) self.expanded_hosts = client.cmd_expandminions( hosts, "test.ping", expr_form=expr_type) self.cdb = RuleStore(self.store_filepath) self.cdb.store_db_init(self.expanded_hosts) self.set_up_and_export_rule_store()