Ejemplo n.º 1
0
def main():
    hosts = None
    action = None
    path = None
    status = None
    dryrun = False
    store_filepath = "/etc/data_retention/dataretention_rules.sq3"

    try:
        (options, remainder) = getopt.gnu_getopt(
            sys.argv[1:], "H:a:p:s:r:dh",
            ["hosts=", "action=", "path=", "status=", "dryrun", "help"])

    except getopt.GetoptError as err:
        usage("Unknown option specified: " + str(err))

    for (opt, val) in options:
        if opt in ["-H", "--hosts"]:
            hosts = val
        elif opt in ["-a", "--action"]:
            action = val
        elif opt in ["-p", "--path"]:
            path = val
        elif opt in ["-s", "--status"]:
            status = val
        elif opt in ["-r", "--rulestore"]:
            store_filepath = val
        elif opt in ["-d", "--dryrun"]:
            dryrun = True
        elif opt in ["-h", "--help"]:
            usage()
        else:
            usage("Unknown option specified: %s" % opt)

    if len(remainder) > 0:
        usage("Unknown option specified: <%s>" % remainder[0])

    check_args(hosts, action, status)

    if not os.path.exists(store_filepath):
        usage('no such rulestore at %s' % store_filepath)

    cdb = RuleStore(store_filepath)
    cdb.store_db_init(None)

    hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts)
    # if we are given one host, check that the host has a table or whine
    if htype == 'glob' and '*' not in hosts:
        if not clouseau.retention.utils.ruleutils.check_host_table_exists(
                cdb, hosts):
            usage('no such host in rule store, %s' % hosts)
    if htype == 'grain' or htype == 'glob':
        client = LocalClientPlus()
        hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype)
    do_action(cdb, action, hosts, status, path, dryrun)
Ejemplo n.º 2
0
def main():
    hosts = None
    action = None
    path = None
    status = None
    dryrun = False
    store_filepath = "/etc/data_retention/dataretention_rules.sq3"

    try:
        (options, remainder) = getopt.gnu_getopt(
            sys.argv[1:], "H:a:p:s:r:dh",
            ["hosts=", "action=", "path=",
             "status=", "dryrun", "help"])

    except getopt.GetoptError as err:
        usage("Unknown option specified: " + str(err))

    for (opt, val) in options:
        if opt in ["-H", "--hosts"]:
            hosts = val
        elif opt in ["-a", "--action"]:
            action = val
        elif opt in ["-p", "--path"]:
            path = val
        elif opt in ["-s", "--status"]:
            status = val
        elif opt in ["-r", "--rulestore"]:
            store_filepath = val
        elif opt in ["-d", "--dryrun"]:
            dryrun = True
        elif opt in ["-h", "--help"]:
            usage()
        else:
            usage("Unknown option specified: %s" % opt)

    if len(remainder) > 0:
        usage("Unknown option specified: <%s>" % remainder[0])

    check_args(hosts, action, status)

    if not os.path.exists(store_filepath):
        usage('no such rulestore at %s' % store_filepath)

    cdb = RuleStore(store_filepath)
    cdb.store_db_init(None)

    hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts)
    # if we are given one host, check that the host has a table or whine
    if htype == 'glob' and '*' not in hosts:
        if not clouseau.retention.utils.ruleutils.check_host_table_exists(cdb, hosts):
            usage('no such host in rule store, %s' % hosts)
    if htype == 'grain' or htype == 'glob':
        client = LocalClientPlus()
        hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype)
    do_action(cdb, action, hosts, status, path, dryrun)
Ejemplo n.º 3
0
    def run_remotely(self):
        '''
        run the current script on specified remote hosts
        '''

        client = LocalClientPlus()

        if self.expanded_hosts is None:
            self.expanded_hosts = client.cmd_expandminions(
                self.hosts, "test.ping", expr_form=self.hosts_expr_type)

        # fixme instead of this we call the right salt module based on the
        # audit type and with the self.auditmodule_args which is a list

        hostbatches = [
            self.
            expanded_hosts[i:i +
                           clouseau.retention.utils.config.conf['batchsize']]
            for i in range(0, len(self.expanded_hosts),
                           clouseau.retention.utils.config.conf['batchsize'])
        ]

        result = {}
        for hosts in hostbatches:
            if self.verbose:
                sys.stderr.write("INFO: running on hosts\n")
                sys.stderr.write(','.join(hosts) + '\n')

            path = os.path.join(os.path.dirname(self.store_filepath),
                                "data_retention.d")
            contents = clouseau.retention.utils.ignores.prep_good_rules_tosend(
                path, hosts)
            if contents:
                new_result = client.cmd_full_return(
                    hosts,
                    'cp.recv',
                    [contents,
                     os.path.join(self.confdir, 'fromstore')],
                    expr_form='list')
            # step two: run the appropriate salt audit module function
            new_result = client.cmd(hosts,
                                    "retentionaudit.%s" %
                                    self.get_auditfunction_name(),
                                    self.auditmodule_args,
                                    expr_form='list',
                                    timeout=self.timeout)

            if new_result is not None:
                result.update(new_result)
            # fixme, collect and report on hosts that did
            # not respond
        return result
Ejemplo n.º 4
0
    def get_salt_known_hosts(self):
        '''
        get all known salt minions by doing a test.ping
        and getting the list of expanded minions (not of responsive hosts,
        of known hosts)
        FIXME there are functions provided for this in more recent
        versions of salt, use them
        '''

        if self.known_hosts is None:
            client = LocalClientPlus()
            # fixme we should just look up the salt keys, cheaper
            self.known_hosts = client.cmd_expandminions('*', "test.ping",
                                                        expr_form='glob')
        return self.known_hosts
Ejemplo n.º 5
0
    def get_salt_known_hosts(self):
        '''
        get all known salt minions by doing a test.ping
        and getting the list of expanded minions (not of responsive hosts,
        of known hosts)
        FIXME there are functions provided for this in more recent
        versions of salt, use them
        '''

        if self.known_hosts is None:
            client = LocalClientPlus()
            # fixme we should just look up the salt keys, cheaper
            self.known_hosts = client.cmd_expandminions('*',
                                                        "test.ping",
                                                        expr_form='glob')
        return self.known_hosts
Ejemplo n.º 6
0
    def run_remotely(self):
        '''
        run the current script on specified remote hosts
        '''

        client = LocalClientPlus()

        if self.expanded_hosts is None:
            self.expanded_hosts = client.cmd_expandminions(
                self.hosts, "test.ping", expr_form=self.hosts_expr_type)

        # fixme instead of this we call the right salt module based on the
        # audit type and with the self.auditmodule_args which is a list

        hostbatches = [self.expanded_hosts[i: i + clouseau.retention.utils.config.conf['batchsize']]
                       for i in range(0, len(self.expanded_hosts),
                                      clouseau.retention.utils.config.conf['batchsize'])]

        result = {}
        for hosts in hostbatches:
            if self.verbose:
                sys.stderr.write("INFO: running on hosts\n")
                sys.stderr.write(','.join(hosts) + '\n')

            path = os.path.join(os.path.dirname(self.store_filepath),
                                "data_retention.d")
            contents = clouseau.retention.utils.ignores.prep_good_rules_tosend(path, hosts)
            if contents:
                new_result = client.cmd_full_return(
                    hosts, 'cp.recv',
                    [contents, os.path.join(self.confdir, 'fromstore')],
                    expr_form='list')
            # step two: run the appropriate salt audit module function
            new_result = client.cmd(
                hosts, "retentionaudit.%s" % self.get_auditfunction_name(),
                self.auditmodule_args,
                expr_form='list', timeout=self.timeout)

            if new_result is not None:
                result.update(new_result)
            # fixme, collect and report on hosts that did
            # not respond
        return result
    def __init__(self,
                 hosts_expr,
                 audit_type,
                 confdir=None,
                 prettyprint=False,
                 show_content=False,
                 dirsizes=False,
                 summary_report=False,
                 depth=2,
                 to_check=None,
                 ignore_also=None,
                 timeout=60,
                 maxfiles=None,
                 store_filepath=None,
                 verbose=False):
        '''
        hosts_expr:   list or grain-based or wildcard expr for hosts
                      to be audited
        audit_type:   type of audit e.g. 'logs', 'homes'
        confdir:      directory where the yaml config files are stored
        prettyprint:  nicely format the output display
        show_content: show the first line or so from problematic files
        dirsizes:     show only directories which have too many files to
                      audit properly, don't report on files at all
        summary_report: do a summary of results instead of detailed
                        this means different thiings depending on the audit
                        type
        depth:        the auditor will give up if a directory has too any files
                      it (saves it form dying on someone's 25gb homedir).
                      this option tells it how far down the tree to go from
                      the top dir of the audit, before starting to count.
                      e.g. do we count in /home/ariel or separately in
                      /home/ariel/* or in /home/ariel/*/*, etc.
        to_check:     comma-separated list of dirs (must end in '/') and/or
                      files that will be checked; if this is None then
                      all dirs/files will be checked
        ignore_also:  comma-separated list of dirs (must end in '/') and/or
                      files that will be skipped in addition to the ones
                      in the config, rules, etc.
        timeout:      salt timeout for running remote commands
        maxfiles:     how many files in a directory tree is too many to audit
                      (at which point we warn about that and move on)
        store_filepath: full path to rule store (sqlite3 db)
        verbose:      show informative messages during processing
        '''

        self.hosts_expr = hosts_expr
        self.audit_type = audit_type
        self.confdir = confdir
        self.locations = audit_type + "_locations"
        self.prettyprint = prettyprint
        self.show_sample_content = show_content
        self.dirsizes = dirsizes
        self.show_summary = summary_report
        self.depth = depth + 1  # actually count of path separators in dirname
        self.to_check = to_check

        self.ignore_also = ignore_also
        self.timeout = timeout
        self.store_filepath = store_filepath
        self.verbose = verbose

        self.max_files = maxfiles
        self.set_up_max_files(maxfiles)

        self.magic = clouseau.retention.utils.magic.magic_open(
            clouseau.retention.utils.magic.MAGIC_NONE)
        self.magic.load()
        self.summary = None
        self.display_from_dict = FileInfo.display_from_dict
        self.runner = None

        if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost":
            # run locally
            self.localaudit = True
            self.expanded_hosts = []
            self.cdb = None
        else:
            self.localaudit = False
            clouseau.retention.utils.config.set_up_conf(confdir)
            client = LocalClientPlus()
            hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr)
            self.expanded_hosts = client.cmd_expandminions(hosts,
                                                           "test.ping",
                                                           expr_form=expr_type)

            self.cdb = RuleStore(self.store_filepath)
            self.cdb.store_db_init(self.expanded_hosts)
            self.set_up_and_export_rule_store()
    def __init__(self, hosts_expr, audit_type,
                 confdir=None,
                 prettyprint=False,
                 show_content=False, dirsizes=False, summary_report=False,
                 depth=2, to_check=None, ignore_also=None,
                 timeout=60, maxfiles=None,
                 store_filepath=None,
                 verbose=False):
        '''
        hosts_expr:   list or grain-based or wildcard expr for hosts
                      to be audited
        audit_type:   type of audit e.g. 'logs', 'homes'
        confdir:      directory where the yaml config files are stored
        prettyprint:  nicely format the output display
        show_content: show the first line or so from problematic files
        dirsizes:     show only directories which have too many files to
                      audit properly, don't report on files at all
        summary_report: do a summary of results instead of detailed
                        this means different thiings depending on the audit
                        type
        depth:        the auditor will give up if a directory has too any files
                      it (saves it form dying on someone's 25gb homedir).
                      this option tells it how far down the tree to go from
                      the top dir of the audit, before starting to count.
                      e.g. do we count in /home/ariel or separately in
                      /home/ariel/* or in /home/ariel/*/*, etc.
        to_check:     comma-separated list of dirs (must end in '/') and/or
                      files that will be checked; if this is None then
                      all dirs/files will be checked
        ignore_also:  comma-separated list of dirs (must end in '/') and/or
                      files that will be skipped in addition to the ones
                      in the config, rules, etc.
        timeout:      salt timeout for running remote commands
        maxfiles:     how many files in a directory tree is too many to audit
                      (at which point we warn about that and move on)
        store_filepath: full path to rule store (sqlite3 db)
        verbose:      show informative messages during processing
        '''

        self.hosts_expr = hosts_expr
        self.audit_type = audit_type
        self.confdir = confdir
        self.locations = audit_type + "_locations"
        self.prettyprint = prettyprint
        self.show_sample_content = show_content
        self.dirsizes = dirsizes
        self.show_summary = summary_report
        self.depth = depth + 1  # actually count of path separators in dirname
        self.to_check = to_check

        self.ignore_also = ignore_also
        self.timeout = timeout
        self.store_filepath = store_filepath
        self.verbose = verbose

        self.max_files = maxfiles
        self.set_up_max_files(maxfiles)

        self.magic = clouseau.retention.utils.magic.magic_open(
            clouseau.retention.utils.magic.MAGIC_NONE)
        self.magic.load()
        self.summary = None
        self.display_from_dict = FileInfo.display_from_dict
        self.runner = None

        if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost":
            # run locally
            self.localaudit = True
            self.expanded_hosts = []
            self.cdb = None
        else:
            self.localaudit = False
            clouseau.retention.utils.config.set_up_conf(confdir)
            client = LocalClientPlus()
            hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr)
            self.expanded_hosts = client.cmd_expandminions(
                hosts, "test.ping", expr_form=expr_type)

            self.cdb = RuleStore(self.store_filepath)
            self.cdb.store_db_init(self.expanded_hosts)
            self.set_up_and_export_rule_store()