def main():
hosts = None
action = None
path = None
status = None
dryrun = False
store_filepath = "/etc/data_retention/dataretention_rules.sq3"
try:
(options, remainder) = getopt.gnu_getopt(
sys.argv[1:], "H:a:p:s:r:dh",
["hosts=", "action=", "path=", "status=", "dryrun", "help"])
except getopt.GetoptError as err:
usage("Unknown option specified: " + str(err))
for (opt, val) in options:
if opt in ["-H", "--hosts"]:
hosts = val
elif opt in ["-a", "--action"]:
action = val
elif opt in ["-p", "--path"]:
path = val
elif opt in ["-s", "--status"]:
status = val
elif opt in ["-r", "--rulestore"]:
store_filepath = val
elif opt in ["-d", "--dryrun"]:
dryrun = True
elif opt in ["-h", "--help"]:
usage()
else:
usage("Unknown option specified: %s" % opt)
if len(remainder) > 0:
usage("Unknown option specified: <%s>" % remainder[0])
check_args(hosts, action, status)
if not os.path.exists(store_filepath):
usage('no such rulestore at %s' % store_filepath)
cdb = RuleStore(store_filepath)
cdb.store_db_init(None)
hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts)
# if we are given one host, check that the host has a table or whine
if htype == 'glob' and '*' not in hosts:
if not clouseau.retention.utils.ruleutils.check_host_table_exists(
cdb, hosts):
usage('no such host in rule store, %s' % hosts)
if htype == 'grain' or htype == 'glob':
client = LocalClientPlus()
hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype)
do_action(cdb, action, hosts, status, path, dryrun)
def main():
hosts = None
action = None
path = None
status = None
dryrun = False
store_filepath = "/etc/data_retention/dataretention_rules.sq3"
try:
(options, remainder) = getopt.gnu_getopt(
sys.argv[1:], "H:a:p:s:r:dh",
["hosts=", "action=", "path=",
"status=", "dryrun", "help"])
except getopt.GetoptError as err:
usage("Unknown option specified: " + str(err))
for (opt, val) in options:
if opt in ["-H", "--hosts"]:
hosts = val
elif opt in ["-a", "--action"]:
action = val
elif opt in ["-p", "--path"]:
path = val
elif opt in ["-s", "--status"]:
status = val
elif opt in ["-r", "--rulestore"]:
store_filepath = val
elif opt in ["-d", "--dryrun"]:
dryrun = True
elif opt in ["-h", "--help"]:
usage()
else:
usage("Unknown option specified: %s" % opt)
if len(remainder) > 0:
usage("Unknown option specified: <%s>" % remainder[0])
check_args(hosts, action, status)
if not os.path.exists(store_filepath):
usage('no such rulestore at %s' % store_filepath)
cdb = RuleStore(store_filepath)
cdb.store_db_init(None)
hosts, htype = clouseau.retention.utils.utils.get_hosts_expr_type(hosts)
# if we are given one host, check that the host has a table or whine
if htype == 'glob' and '*' not in hosts:
if not clouseau.retention.utils.ruleutils.check_host_table_exists(cdb, hosts):
usage('no such host in rule store, %s' % hosts)
if htype == 'grain' or htype == 'glob':
client = LocalClientPlus()
hosts = client.cmd_expandminions(hosts, "test.ping", expr_form=htype)
do_action(cdb, action, hosts, status, path, dryrun)
def run_remotely(self):
'''
run the current script on specified remote hosts
'''
client = LocalClientPlus()
if self.expanded_hosts is None:
self.expanded_hosts = client.cmd_expandminions(
self.hosts, "test.ping", expr_form=self.hosts_expr_type)
# fixme instead of this we call the right salt module based on the
# audit type and with the self.auditmodule_args which is a list
hostbatches = [
self.
expanded_hosts[i:i +
clouseau.retention.utils.config.conf['batchsize']]
for i in range(0, len(self.expanded_hosts),
clouseau.retention.utils.config.conf['batchsize'])
]
result = {}
for hosts in hostbatches:
if self.verbose:
sys.stderr.write("INFO: running on hosts\n")
sys.stderr.write(','.join(hosts) + '\n')
path = os.path.join(os.path.dirname(self.store_filepath),
"data_retention.d")
contents = clouseau.retention.utils.ignores.prep_good_rules_tosend(
path, hosts)
if contents:
new_result = client.cmd_full_return(
hosts,
'cp.recv',
[contents,
os.path.join(self.confdir, 'fromstore')],
expr_form='list')
# step two: run the appropriate salt audit module function
new_result = client.cmd(hosts,
"retentionaudit.%s" %
self.get_auditfunction_name(),
self.auditmodule_args,
expr_form='list',
timeout=self.timeout)
if new_result is not None:
result.update(new_result)
# fixme, collect and report on hosts that did
# not respond
return result
def get_salt_known_hosts(self):
'''
get all known salt minions by doing a test.ping
and getting the list of expanded minions (not of responsive hosts,
of known hosts)
FIXME there are functions provided for this in more recent
versions of salt, use them
'''
if self.known_hosts is None:
client = LocalClientPlus()
# fixme we should just look up the salt keys, cheaper
self.known_hosts = client.cmd_expandminions('*', "test.ping",
expr_form='glob')
return self.known_hosts
def get_salt_known_hosts(self):
'''
get all known salt minions by doing a test.ping
and getting the list of expanded minions (not of responsive hosts,
of known hosts)
FIXME there are functions provided for this in more recent
versions of salt, use them
'''
if self.known_hosts is None:
client = LocalClientPlus()
# fixme we should just look up the salt keys, cheaper
self.known_hosts = client.cmd_expandminions('*',
"test.ping",
expr_form='glob')
return self.known_hosts
def run_remotely(self):
'''
run the current script on specified remote hosts
'''
client = LocalClientPlus()
if self.expanded_hosts is None:
self.expanded_hosts = client.cmd_expandminions(
self.hosts, "test.ping", expr_form=self.hosts_expr_type)
# fixme instead of this we call the right salt module based on the
# audit type and with the self.auditmodule_args which is a list
hostbatches = [self.expanded_hosts[i: i + clouseau.retention.utils.config.conf['batchsize']]
for i in range(0, len(self.expanded_hosts),
clouseau.retention.utils.config.conf['batchsize'])]
result = {}
for hosts in hostbatches:
if self.verbose:
sys.stderr.write("INFO: running on hosts\n")
sys.stderr.write(','.join(hosts) + '\n')
path = os.path.join(os.path.dirname(self.store_filepath),
"data_retention.d")
contents = clouseau.retention.utils.ignores.prep_good_rules_tosend(path, hosts)
if contents:
new_result = client.cmd_full_return(
hosts, 'cp.recv',
[contents, os.path.join(self.confdir, 'fromstore')],
expr_form='list')
# step two: run the appropriate salt audit module function
new_result = client.cmd(
hosts, "retentionaudit.%s" % self.get_auditfunction_name(),
self.auditmodule_args,
expr_form='list', timeout=self.timeout)
if new_result is not None:
result.update(new_result)
# fixme, collect and report on hosts that did
# not respond
return result
def __init__(self,
hosts_expr,
audit_type,
confdir=None,
prettyprint=False,
show_content=False,
dirsizes=False,
summary_report=False,
depth=2,
to_check=None,
ignore_also=None,
timeout=60,
maxfiles=None,
store_filepath=None,
verbose=False):
'''
hosts_expr: list or grain-based or wildcard expr for hosts
to be audited
audit_type: type of audit e.g. 'logs', 'homes'
confdir: directory where the yaml config files are stored
prettyprint: nicely format the output display
show_content: show the first line or so from problematic files
dirsizes: show only directories which have too many files to
audit properly, don't report on files at all
summary_report: do a summary of results instead of detailed
this means different thiings depending on the audit
type
depth: the auditor will give up if a directory has too any files
it (saves it form dying on someone's 25gb homedir).
this option tells it how far down the tree to go from
the top dir of the audit, before starting to count.
e.g. do we count in /home/ariel or separately in
/home/ariel/* or in /home/ariel/*/*, etc.
to_check: comma-separated list of dirs (must end in '/') and/or
files that will be checked; if this is None then
all dirs/files will be checked
ignore_also: comma-separated list of dirs (must end in '/') and/or
files that will be skipped in addition to the ones
in the config, rules, etc.
timeout: salt timeout for running remote commands
maxfiles: how many files in a directory tree is too many to audit
(at which point we warn about that and move on)
store_filepath: full path to rule store (sqlite3 db)
verbose: show informative messages during processing
'''
self.hosts_expr = hosts_expr
self.audit_type = audit_type
self.confdir = confdir
self.locations = audit_type + "_locations"
self.prettyprint = prettyprint
self.show_sample_content = show_content
self.dirsizes = dirsizes
self.show_summary = summary_report
self.depth = depth + 1 # actually count of path separators in dirname
self.to_check = to_check
self.ignore_also = ignore_also
self.timeout = timeout
self.store_filepath = store_filepath
self.verbose = verbose
self.max_files = maxfiles
self.set_up_max_files(maxfiles)
self.magic = clouseau.retention.utils.magic.magic_open(
clouseau.retention.utils.magic.MAGIC_NONE)
self.magic.load()
self.summary = None
self.display_from_dict = FileInfo.display_from_dict
self.runner = None
if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost":
# run locally
self.localaudit = True
self.expanded_hosts = []
self.cdb = None
else:
self.localaudit = False
clouseau.retention.utils.config.set_up_conf(confdir)
client = LocalClientPlus()
hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr)
self.expanded_hosts = client.cmd_expandminions(hosts,
"test.ping",
expr_form=expr_type)
self.cdb = RuleStore(self.store_filepath)
self.cdb.store_db_init(self.expanded_hosts)
self.set_up_and_export_rule_store()
def __init__(self, hosts_expr, audit_type,
confdir=None,
prettyprint=False,
show_content=False, dirsizes=False, summary_report=False,
depth=2, to_check=None, ignore_also=None,
timeout=60, maxfiles=None,
store_filepath=None,
verbose=False):
'''
hosts_expr: list or grain-based or wildcard expr for hosts
to be audited
audit_type: type of audit e.g. 'logs', 'homes'
confdir: directory where the yaml config files are stored
prettyprint: nicely format the output display
show_content: show the first line or so from problematic files
dirsizes: show only directories which have too many files to
audit properly, don't report on files at all
summary_report: do a summary of results instead of detailed
this means different thiings depending on the audit
type
depth: the auditor will give up if a directory has too any files
it (saves it form dying on someone's 25gb homedir).
this option tells it how far down the tree to go from
the top dir of the audit, before starting to count.
e.g. do we count in /home/ariel or separately in
/home/ariel/* or in /home/ariel/*/*, etc.
to_check: comma-separated list of dirs (must end in '/') and/or
files that will be checked; if this is None then
all dirs/files will be checked
ignore_also: comma-separated list of dirs (must end in '/') and/or
files that will be skipped in addition to the ones
in the config, rules, etc.
timeout: salt timeout for running remote commands
maxfiles: how many files in a directory tree is too many to audit
(at which point we warn about that and move on)
store_filepath: full path to rule store (sqlite3 db)
verbose: show informative messages during processing
'''
self.hosts_expr = hosts_expr
self.audit_type = audit_type
self.confdir = confdir
self.locations = audit_type + "_locations"
self.prettyprint = prettyprint
self.show_sample_content = show_content
self.dirsizes = dirsizes
self.show_summary = summary_report
self.depth = depth + 1 # actually count of path separators in dirname
self.to_check = to_check
self.ignore_also = ignore_also
self.timeout = timeout
self.store_filepath = store_filepath
self.verbose = verbose
self.max_files = maxfiles
self.set_up_max_files(maxfiles)
self.magic = clouseau.retention.utils.magic.magic_open(
clouseau.retention.utils.magic.MAGIC_NONE)
self.magic.load()
self.summary = None
self.display_from_dict = FileInfo.display_from_dict
self.runner = None
if self.hosts_expr == "127.0.0.1" or self.hosts_expr == "localhost":
# run locally
self.localaudit = True
self.expanded_hosts = []
self.cdb = None
else:
self.localaudit = False
clouseau.retention.utils.config.set_up_conf(confdir)
client = LocalClientPlus()
hosts, expr_type = Runner.get_hosts_expr_type(self.hosts_expr)
self.expanded_hosts = client.cmd_expandminions(
hosts, "test.ping", expr_form=expr_type)
self.cdb = RuleStore(self.store_filepath)
self.cdb.store_db_init(self.expanded_hosts)
self.set_up_and_export_rule_store()