def update_status_rules_from_report(self, report):
hostlist = report.keys()
for host in hostlist:
try:
problem_rules = clouseau.retention.utils.ruleutils.get_rules(
self.cdb, host, Status.text_to_status('problem'))
except:
print 'WARNING: problem retrieving problem rules for host', host
problem_rules = None
if problem_rules is not None:
existing_problems = [rule['path'] for rule in problem_rules]
else:
existing_problems = []
dirs_problem, dirs_skipped = get_dirs_toexamine(report[host])
if dirs_problem is not None:
dirs_problem = list(set(dirs_problem))
for dirname in dirs_problem:
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, dirname,
clouseau.retention.utils.ruleutils.text_to_entrytype('dir'),
Status.text_to_status('problem'), host)
if dirs_skipped is not None:
dirs_skipped = list(set(dirs_skipped))
for dirname in dirs_skipped:
if dirname in dirs_problem or dirname in existing_problems:
# problem report overrides 'too many to audit'
continue
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, dirname,
clouseau.retention.utils.ruleutils.text_to_entrytype('dir'),
Status.text_to_status('unreviewed'), host)
def show_menu(self, level):
if level == 'top':
text = ("S(set status)/E(examine directory)/"
"Filter directory listings/"
"I(ignore)/R(manage rules)/Q(quit menu)")
command = self.get_menu_entry(['S', 'E', 'I', 'F', 'R', 'Q'], 'S', text)
elif level == 'status':
text = Status.get_statuses_prompt(", ") + ", Q(quit status menu)"
command = self.get_menu_entry(Status.STATUSES + ['Q'], text,
Status.text_to_status('good'))
if command == 'Q' or command == 'q':
level = 'top'
elif level == 'examine':
text = ("D(down a level)/U(up a level)/E(show entries)/"
"C(show contents of file)/R(show rules)/"
"F(filter directory listings/"
"M(mark file(s))/Q(quit examine menu)")
command = self.get_menu_entry(['D', 'U', 'E', 'F', 'C', 'R', 'M', 'Q'], 'E', text)
if command == 'Q' or command == 'q':
level = 'top'
elif level == 'rule':
text = ("S(show all rules of type)/D(show rules covering dir)/"
"C(show rules covering dir contents)/"
"A(add rule to rules store)/"
"R(remove rule from rules store/"
"E(export rules from store to file)/"
"I(import rules from file to store)/Q(quit rule menu)")
command = self.get_menu_entry(['S', 'C', 'A', 'R', 'E', 'I', 'Q'], 'D', text)
if command == 'Q' or command == 'q':
level = 'top'
else:
command = None
return command
def do_show_rules_with_status(self):
default = Status.text_to_status('problem')
self.cmpl.set_choices_completion(['A'] + Status.STATUSES + ['Q'], default)
while True:
statuses_text = Status.get_statuses_prompt(", ")
status = raw_input("status type A(all), " + statuses_text +
", Q(quit)) [%s]: " % default)
status = status.strip()
if status == "":
status = default
if status == 'q' or status == 'Q':
return None
elif status[0].upper() not in ['A'] + Status.STATUSES:
print "Unknown status type"
continue
readline.set_completer(None)
prefix = raw_input("starting with prefix? [/]: ")
prefix = prefix.strip()
if prefix == "":
prefix = "/"
if status == 'a' or status == 'A':
clouseau.retention.utils.ruleutils.show_rules(
self.cdb, self.cenv.host, prefix=prefix)
return True
elif status[0].upper() in Status.STATUSES:
clouseau.retention.utils.ruleutils.show_rules(
self.cdb, self.cenv.host, status[0].upper(), prefix=prefix)
return True
def update_status_rules_from_report(self, report):
hostlist = report.keys()
for host in hostlist:
try:
problem_rules = clouseau.retention.utils.ruleutils.get_rules(
self.cdb, host, Status.text_to_status('problem'))
except:
print 'WARNING: problem retrieving problem rules for host', host
problem_rules = None
if problem_rules is not None:
existing_problems = [rule['path'] for rule in problem_rules]
else:
existing_problems = []
dirs_problem, dirs_skipped = get_dirs_toexamine(report[host])
if dirs_problem is not None:
dirs_problem = list(set(dirs_problem))
for dirname in dirs_problem:
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, dirname,
clouseau.retention.utils.ruleutils.text_to_entrytype(
'dir'), Status.text_to_status('problem'), host)
if dirs_skipped is not None:
dirs_skipped = list(set(dirs_skipped))
for dirname in dirs_skipped:
if dirname in dirs_problem or dirname in existing_problems:
# problem report overrides 'too many to audit'
continue
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, dirname,
clouseau.retention.utils.ruleutils.text_to_entrytype(
'dir'), Status.text_to_status('unreviewed'), host)
def do_show_rules_with_status(self):
default = Status.text_to_status('problem')
self.cmpl.set_choices_completion(['A'] + Status.STATUSES + ['Q'],
default)
while True:
statuses_text = Status.get_statuses_prompt(", ")
status = raw_input("status type A(all), " + statuses_text +
", Q(quit)) [%s]: " % default)
status = status.strip()
if status == "":
status = default
if status == 'q' or status == 'Q':
return None
elif status[0].upper() not in ['A'] + Status.STATUSES:
print "Unknown status type"
continue
readline.set_completer(None)
prefix = raw_input("starting with prefix? [/]: ")
prefix = prefix.strip()
if prefix == "":
prefix = "/"
if status == 'a' or status == 'A':
clouseau.retention.utils.ruleutils.show_rules(self.cdb,
self.cenv.host,
prefix=prefix)
return True
elif status[0].upper() in Status.STATUSES:
clouseau.retention.utils.ruleutils.show_rules(
self.cdb, self.cenv.host, status[0].upper(), prefix=prefix)
return True
def do_add_rule(self):
# fixme need different completer here I think, that
# completes relative to self.cwdir
readline.set_completer(None)
path = raw_input("path or wildcard expr in rule (empty to quit): ")
path = path.strip()
if path == '':
return True
default = Status.text_to_status('good')
self.cmpl.set_choices_completion(Status.STATUSES + ['Q'], default)
while True:
statuses_text = Status.get_statuses_prompt(", ")
status = raw_input(statuses_text + " Q(quit)) [%s]: " % default)
status = status.strip()
if status == "":
status = default
if status[0].upper() in Status.STATUSES:
status = status[0].upper()
break
elif status == 'q' or status == 'Q':
return None
else:
print "Unknown status type"
continue
# fixme should check that any wildcard is only one and only
# in the last component... someday
if path[0] != os.path.sep:
path = os.path.join(self.cenv.cwdir, path)
if path[-1] == os.path.sep:
path = path[:-1]
filetype = clouseau.retention.utils.ruleutils.text_to_entrytype(
'dir')
else:
filetype = clouseau.retention.utils.ruleutils.text_to_entrytype(
'file')
clouseau.retention.utils.ruleutils.do_add_rule(self.cdb, path,
filetype, status,
self.cenv.host)
# update the ignores list since we have a new rule
results = clouseau.retention.utils.ignores.get_ignored_from_rulestore(
self.cdb, [self.cenv.host])
if self.cenv.host in results:
self.ignored_from_rulestore[self.cenv.host] = results[
self.cenv.host]
return True
def do_mark(self):
readline.set_completer(self.cmpl.dir_entries_completion)
file_expr = raw_input("file or dirname expression (empty to quit): ")
file_expr = file_expr.strip()
if file_expr == '':
return True
if file_expr[-1] == os.path.sep:
file_expr = file_expr[:-1]
if '*' in file_expr:
entries_todo = self.get_entries_from_wildcard(file_expr)
else:
entries_todo = [file_expr]
self.dircontents.get(self.cenv.host, self.cenv.cwdir, self.batchno)
if not self.dircontents.entries:
print 'failed to get directory contents for', self.cenv.cwdir
print 'marking dirs/files regardless'
for entry in entries_todo:
if entry not in self.dircontents.entries_dict:
print 'skipping %s, not in current dir listing' % entry
print self.dircontents.entries_dict
continue
filetype = clouseau.retention.utils.ruleutils.entrytype_to_text(
self.dircontents.entries_dict[entry]['type'])
if filetype == 'link':
print 'No need to mark', file_expr, 'links are always skipped'
continue
elif filetype != 'dir' and filetype != 'file':
print 'Not a dir or regular file, no need to mark, skipping'
continue
status = Status.text_to_status('good')
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, file_expr, filetype, status, self.cenv.host)
return True
def do_mark(self):
readline.set_completer(self.cmpl.dir_entries_completion)
file_expr = raw_input("file or dirname expression (empty to quit): ")
file_expr = file_expr.strip()
if file_expr == '':
return True
if file_expr[-1] == os.path.sep:
file_expr = file_expr[:-1]
if '*' in file_expr:
entries_todo = self.get_entries_from_wildcard(file_expr)
else:
entries_todo = [file_expr]
self.dircontents.get(self.cenv.host, self.cenv.cwdir, self.batchno)
if not self.dircontents.entries:
print 'failed to get directory contents for', self.cenv.cwdir
print 'marking dirs/files regardless'
for entry in entries_todo:
if entry not in self.dircontents.entries_dict:
print 'skipping %s, not in current dir listing' % entry
print self.dircontents.entries_dict
continue
filetype = clouseau.retention.utils.ruleutils.entrytype_to_text(
self.dircontents.entries_dict[entry]['type'])
if filetype == 'link':
print 'No need to mark', file_expr, 'links are always skipped'
continue
elif filetype != 'dir' and filetype != 'file':
print 'Not a dir or regular file, no need to mark, skipping'
continue
status = Status.text_to_status('good')
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, file_expr, filetype, status, self.cenv.host)
return True
def do_add_rule(self):
# fixme need different completer here I think, that
# completes relative to self.cwdir
readline.set_completer(None)
path = raw_input("path or wildcard expr in rule (empty to quit): ")
path = path.strip()
if path == '':
return True
default = Status.text_to_status('good')
self.cmpl.set_choices_completion(Status.STATUSES + ['Q'], default)
while True:
statuses_text = Status.get_statuses_prompt(", ")
status = raw_input(statuses_text + " Q(quit)) [%s]: " %
default)
status = status.strip()
if status == "":
status = default
if status[0].upper() in Status.STATUSES:
status = status[0].upper()
break
elif status == 'q' or status == 'Q':
return None
else:
print "Unknown status type"
continue
# fixme should check that any wildcard is only one and only
# in the last component... someday
if path[0] != os.path.sep:
path = os.path.join(self.cenv.cwdir, path)
if path[-1] == os.path.sep:
path = path[:-1]
filetype = clouseau.retention.utils.ruleutils.text_to_entrytype('dir')
else:
filetype = clouseau.retention.utils.ruleutils.text_to_entrytype('file')
clouseau.retention.utils.ruleutils.do_add_rule(
self.cdb, path, filetype, status, self.cenv.host)
# update the ignores list since we have a new rule
results = clouseau.retention.utils.ignores.get_ignored_from_rulestore(
self.cdb, [self.cenv.host])
if self.cenv.host in results:
self.ignored_from_rulestore[self.cenv.host] = results[self.cenv.host]
return True
def row_to_rule(row):
# ('/home/ariel/wmf/security', '/home/ariel/wmf/security/openjdk6', 'D', 'G')
(basedir, name, entrytype, status) = row
basedir = clouseau.retention.utils.rule.from_unicode(basedir)
name = clouseau.retention.utils.rule.from_unicode(name)
rule = {'path': os.path.join(basedir, name),
'type': entrytype_to_text(entrytype),
'status': Status.status_to_text(status)}
return rule
def row_to_rule(row):
# ('/home/ariel/wmf/security', '/home/ariel/wmf/security/openjdk6', 'D', 'G')
(basedir, name, entrytype, status) = row
basedir = clouseau.retention.utils.rule.from_unicode(basedir)
name = clouseau.retention.utils.rule.from_unicode(name)
rule = {
'path': os.path.join(basedir, name),
'type': entrytype_to_text(entrytype),
'status': Status.status_to_text(status)
}
return rule
def set_up_and_export_rule_store(self):
hosts = self.cdb.store_db_list_all_hosts()
destdir = os.path.join(os.path.dirname(self.store_filepath),
"data_retention.d")
if not os.path.isdir(destdir):
os.makedirs(destdir, 0755)
for host in hosts:
all_destpath = os.path.join(destdir, host + "_store.yaml")
clouseau.retention.utils.ruleutils.export_rules(self.cdb, all_destpath, host)
good_destpath = os.path.join(destdir, host + "_store_good.yaml")
clouseau.retention.utils.ruleutils.export_rules(self.cdb, good_destpath, host,
Status.text_to_status('good'))
def set_up_and_export_rule_store(self):
hosts = self.cdb.store_db_list_all_hosts()
destdir = os.path.join(os.path.dirname(self.store_filepath),
"data_retention.d")
if not os.path.isdir(destdir):
os.makedirs(destdir, 0755)
for host in hosts:
all_destpath = os.path.join(destdir, host + "_store.yaml")
clouseau.retention.utils.ruleutils.export_rules(
self.cdb, all_destpath, host)
good_destpath = os.path.join(destdir, host + "_store_good.yaml")
clouseau.retention.utils.ruleutils.export_rules(
self.cdb, good_destpath, host, Status.text_to_status('good'))
def check_args(hosts, action, status):
if hosts is None:
usage("Mandatory 'hosts' argument not specified")
if action is None:
usage("Mandatory 'action' argument not specified")
if action not in ['show', 'delete', 'add']:
usage('unknown action %s specified' % action)
if status is not None:
status = Status.status_to_text(status)
if status is None:
usage('unknown status %s specified' % status)
def check_args(hosts, action, status):
if hosts is None:
usage("Mandatory 'hosts' argument not specified")
if action is None:
usage("Mandatory 'action' argument not specified")
if action not in ['show', 'delete', 'add']:
usage('unknown action %s specified' % action)
if status is not None:
status = Status.status_to_text(status)
if status is None:
usage('unknown status %s specified' % status)
def show_menu(self, level):
if level == 'top':
text = ("S(set status)/E(examine directory)/"
"Filter directory listings/"
"I(ignore)/R(manage rules)/Q(quit menu)")
command = self.get_menu_entry(['S', 'E', 'I', 'F', 'R', 'Q'], 'S',
text)
elif level == 'status':
text = Status.get_statuses_prompt(", ") + ", Q(quit status menu)"
command = self.get_menu_entry(Status.STATUSES + ['Q'], text,
Status.text_to_status('good'))
if command == 'Q' or command == 'q':
level = 'top'
elif level == 'examine':
text = ("D(down a level)/U(up a level)/E(show entries)/"
"C(show contents of file)/R(show rules)/"
"F(filter directory listings/"
"M(mark file(s))/Q(quit examine menu)")
command = self.get_menu_entry(
['D', 'U', 'E', 'F', 'C', 'R', 'M', 'Q'], 'E', text)
if command == 'Q' or command == 'q':
level = 'top'
elif level == 'rule':
text = ("S(show all rules of type)/D(show rules covering dir)/"
"C(show rules covering dir contents)/"
"A(add rule to rules store)/"
"R(remove rule from rules store/"
"E(export rules from store to file)/"
"I(import rules from file to store)/Q(quit rule menu)")
command = self.get_menu_entry(['S', 'C', 'A', 'R', 'E', 'I', 'Q'],
'D', text)
if command == 'Q' or command == 'q':
level = 'top'
else:
command = None
return command
def get_ignored_from_rulestore(cdb, hosts):
ignored = {}
for host in hosts:
rules_from_store = clouseau.retention.utils.ruleutils.get_rules(
cdb, host, Status.text_to_status('good'))
if rules_from_store is not None:
ignored[host] = clouseau.retention.utils.ignores.init_ignored()
for rule in rules_from_store:
if rule['status'] != 'good':
continue
path = rule['path']
if rule['type'] == 'dir':
if path.endswith('/'):
path = path[:-1]
ignored[host]['dirs']['/'].append(path)
else:
ignored[host]['files']['/'].append(path)
return ignored
def get_ignored_from_rulestore(cdb, hosts):
ignored = {}
for host in hosts:
rules_from_store = clouseau.retention.utils.ruleutils.get_rules(
cdb, host, Status.text_to_status('good'))
if rules_from_store is not None:
ignored[host] = clouseau.retention.utils.ignores.init_ignored()
for rule in rules_from_store:
if rule['status'] != 'good':
continue
path = rule['path']
if rule['type'] == 'dir':
if path.endswith('/'):
path = path[:-1]
ignored[host]['dirs']['/'].append(path)
else:
ignored[host]['files']['/'].append(path)
return ignored