def update_status_rules_from_report(self, report):
        hostlist = report.keys()
        for host in hostlist:
            try:
                problem_rules = clouseau.retention.utils.ruleutils.get_rules(
                    self.cdb, host, Status.text_to_status('problem'))
            except:
                print 'WARNING: problem retrieving problem rules for host', host
                problem_rules = None
            if problem_rules is not None:
                existing_problems = [rule['path'] for rule in problem_rules]
            else:
                existing_problems = []

            dirs_problem, dirs_skipped = get_dirs_toexamine(report[host])
            if dirs_problem is not None:
                dirs_problem = list(set(dirs_problem))
                for dirname in dirs_problem:
                    clouseau.retention.utils.ruleutils.do_add_rule(
                        self.cdb, dirname,
                        clouseau.retention.utils.ruleutils.text_to_entrytype('dir'),
                        Status.text_to_status('problem'), host)

            if dirs_skipped is not None:
                dirs_skipped = list(set(dirs_skipped))
                for dirname in dirs_skipped:
                    if dirname in dirs_problem or dirname in existing_problems:
                        # problem report overrides 'too many to audit'
                        continue
                    clouseau.retention.utils.ruleutils.do_add_rule(
                        self.cdb, dirname,
                        clouseau.retention.utils.ruleutils.text_to_entrytype('dir'),
                        Status.text_to_status('unreviewed'), host)
Beispiel #2
0
 def show_menu(self, level):
     if level == 'top':
         text = ("S(set status)/E(examine directory)/"
                 "Filter directory listings/"
                 "I(ignore)/R(manage rules)/Q(quit menu)")
         command = self.get_menu_entry(['S', 'E', 'I', 'F', 'R', 'Q'], 'S', text)
     elif level == 'status':
         text = Status.get_statuses_prompt(", ") + ", Q(quit status menu)"
         command = self.get_menu_entry(Status.STATUSES + ['Q'], text,
                                       Status.text_to_status('good'))
         if command == 'Q' or command == 'q':
             level = 'top'
     elif level == 'examine':
         text = ("D(down a level)/U(up a level)/E(show entries)/"
                 "C(show contents of file)/R(show rules)/"
                 "F(filter directory listings/"
                 "M(mark file(s))/Q(quit examine menu)")
         command = self.get_menu_entry(['D', 'U', 'E', 'F', 'C', 'R', 'M', 'Q'], 'E', text)
         if command == 'Q' or command == 'q':
             level = 'top'
     elif level == 'rule':
         text = ("S(show all rules of type)/D(show rules covering dir)/"
                 "C(show rules covering dir contents)/"
                 "A(add rule to rules store)/"
                 "R(remove rule from rules store/"
                 "E(export rules from store to file)/"
                 "I(import rules from file to store)/Q(quit rule menu)")
         command = self.get_menu_entry(['S', 'C', 'A', 'R', 'E', 'I', 'Q'], 'D', text)
         if command == 'Q' or command == 'q':
             level = 'top'
     else:
         command = None
     return command
Beispiel #3
0
    def do_show_rules_with_status(self):
        default = Status.text_to_status('problem')
        self.cmpl.set_choices_completion(['A'] + Status.STATUSES + ['Q'], default)
        while True:
            statuses_text = Status.get_statuses_prompt(", ")
            status = raw_input("status type A(all), " + statuses_text +
                               ", Q(quit)) [%s]: " % default)
            status = status.strip()
            if status == "":
                status = default

            if status == 'q' or status == 'Q':
                return None
            elif status[0].upper() not in ['A'] + Status.STATUSES:
                print "Unknown status type"
                continue

            readline.set_completer(None)
            prefix = raw_input("starting with prefix? [/]: ")
            prefix = prefix.strip()
            if prefix == "":
                prefix = "/"
            if status == 'a' or status == 'A':
                clouseau.retention.utils.ruleutils.show_rules(
                    self.cdb, self.cenv.host, prefix=prefix)
                return True
            elif status[0].upper() in Status.STATUSES:
                clouseau.retention.utils.ruleutils.show_rules(
                    self.cdb, self.cenv.host, status[0].upper(), prefix=prefix)
                return True
    def update_status_rules_from_report(self, report):
        hostlist = report.keys()
        for host in hostlist:
            try:
                problem_rules = clouseau.retention.utils.ruleutils.get_rules(
                    self.cdb, host, Status.text_to_status('problem'))
            except:
                print 'WARNING: problem retrieving problem rules for host', host
                problem_rules = None
            if problem_rules is not None:
                existing_problems = [rule['path'] for rule in problem_rules]
            else:
                existing_problems = []

            dirs_problem, dirs_skipped = get_dirs_toexamine(report[host])
            if dirs_problem is not None:
                dirs_problem = list(set(dirs_problem))
                for dirname in dirs_problem:
                    clouseau.retention.utils.ruleutils.do_add_rule(
                        self.cdb, dirname,
                        clouseau.retention.utils.ruleutils.text_to_entrytype(
                            'dir'), Status.text_to_status('problem'), host)

            if dirs_skipped is not None:
                dirs_skipped = list(set(dirs_skipped))
                for dirname in dirs_skipped:
                    if dirname in dirs_problem or dirname in existing_problems:
                        # problem report overrides 'too many to audit'
                        continue
                    clouseau.retention.utils.ruleutils.do_add_rule(
                        self.cdb, dirname,
                        clouseau.retention.utils.ruleutils.text_to_entrytype(
                            'dir'), Status.text_to_status('unreviewed'), host)
Beispiel #5
0
    def do_show_rules_with_status(self):
        default = Status.text_to_status('problem')
        self.cmpl.set_choices_completion(['A'] + Status.STATUSES + ['Q'],
                                         default)
        while True:
            statuses_text = Status.get_statuses_prompt(", ")
            status = raw_input("status type A(all), " + statuses_text +
                               ", Q(quit)) [%s]: " % default)
            status = status.strip()
            if status == "":
                status = default

            if status == 'q' or status == 'Q':
                return None
            elif status[0].upper() not in ['A'] + Status.STATUSES:
                print "Unknown status type"
                continue

            readline.set_completer(None)
            prefix = raw_input("starting with prefix? [/]: ")
            prefix = prefix.strip()
            if prefix == "":
                prefix = "/"
            if status == 'a' or status == 'A':
                clouseau.retention.utils.ruleutils.show_rules(self.cdb,
                                                              self.cenv.host,
                                                              prefix=prefix)
                return True
            elif status[0].upper() in Status.STATUSES:
                clouseau.retention.utils.ruleutils.show_rules(
                    self.cdb, self.cenv.host, status[0].upper(), prefix=prefix)
                return True
Beispiel #6
0
    def do_add_rule(self):
        # fixme need different completer here I think, that
        # completes relative to self.cwdir
        readline.set_completer(None)
        path = raw_input("path or wildcard expr in rule (empty to quit): ")
        path = path.strip()
        if path == '':
            return True
        default = Status.text_to_status('good')
        self.cmpl.set_choices_completion(Status.STATUSES + ['Q'], default)
        while True:
            statuses_text = Status.get_statuses_prompt(", ")
            status = raw_input(statuses_text + " Q(quit)) [%s]: " % default)
            status = status.strip()
            if status == "":
                status = default
            if status[0].upper() in Status.STATUSES:
                status = status[0].upper()
                break
            elif status == 'q' or status == 'Q':
                return None
            else:
                print "Unknown status type"
                continue

        # fixme should check that any wildcard is only one and only
        # in the last component... someday

        if path[0] != os.path.sep:
            path = os.path.join(self.cenv.cwdir, path)
        if path[-1] == os.path.sep:
            path = path[:-1]
            filetype = clouseau.retention.utils.ruleutils.text_to_entrytype(
                'dir')
        else:
            filetype = clouseau.retention.utils.ruleutils.text_to_entrytype(
                'file')

        clouseau.retention.utils.ruleutils.do_add_rule(self.cdb, path,
                                                       filetype, status,
                                                       self.cenv.host)
        # update the ignores list since we have a new rule
        results = clouseau.retention.utils.ignores.get_ignored_from_rulestore(
            self.cdb, [self.cenv.host])
        if self.cenv.host in results:
            self.ignored_from_rulestore[self.cenv.host] = results[
                self.cenv.host]
        return True
Beispiel #7
0
 def do_mark(self):
     readline.set_completer(self.cmpl.dir_entries_completion)
     file_expr = raw_input("file or dirname expression (empty to quit): ")
     file_expr = file_expr.strip()
     if file_expr == '':
         return True
     if file_expr[-1] == os.path.sep:
         file_expr = file_expr[:-1]
     if '*' in file_expr:
         entries_todo = self.get_entries_from_wildcard(file_expr)
     else:
         entries_todo = [file_expr]
         self.dircontents.get(self.cenv.host, self.cenv.cwdir, self.batchno)
         if not self.dircontents.entries:
             print 'failed to get directory contents for', self.cenv.cwdir
             print 'marking dirs/files regardless'
     for entry in entries_todo:
         if entry not in self.dircontents.entries_dict:
             print 'skipping %s, not in current dir listing' % entry
             print self.dircontents.entries_dict
             continue
         filetype = clouseau.retention.utils.ruleutils.entrytype_to_text(
             self.dircontents.entries_dict[entry]['type'])
         if filetype == 'link':
             print 'No need to mark', file_expr, 'links are always skipped'
             continue
         elif filetype != 'dir' and filetype != 'file':
             print 'Not a dir or regular file, no need to mark, skipping'
             continue
         status = Status.text_to_status('good')
         clouseau.retention.utils.ruleutils.do_add_rule(
             self.cdb, file_expr, filetype, status, self.cenv.host)
     return True
Beispiel #8
0
 def do_mark(self):
     readline.set_completer(self.cmpl.dir_entries_completion)
     file_expr = raw_input("file or dirname expression (empty to quit): ")
     file_expr = file_expr.strip()
     if file_expr == '':
         return True
     if file_expr[-1] == os.path.sep:
         file_expr = file_expr[:-1]
     if '*' in file_expr:
         entries_todo = self.get_entries_from_wildcard(file_expr)
     else:
         entries_todo = [file_expr]
         self.dircontents.get(self.cenv.host, self.cenv.cwdir, self.batchno)
         if not self.dircontents.entries:
             print 'failed to get directory contents for', self.cenv.cwdir
             print 'marking dirs/files regardless'
     for entry in entries_todo:
         if entry not in self.dircontents.entries_dict:
             print 'skipping %s, not in current dir listing' % entry
             print self.dircontents.entries_dict
             continue
         filetype = clouseau.retention.utils.ruleutils.entrytype_to_text(
             self.dircontents.entries_dict[entry]['type'])
         if filetype == 'link':
             print 'No need to mark', file_expr, 'links are always skipped'
             continue
         elif filetype != 'dir' and filetype != 'file':
             print 'Not a dir or regular file, no need to mark, skipping'
             continue
         status = Status.text_to_status('good')
         clouseau.retention.utils.ruleutils.do_add_rule(
             self.cdb, file_expr, filetype, status, self.cenv.host)
     return True
Beispiel #9
0
    def do_add_rule(self):
        # fixme need different completer here I think, that
        # completes relative to self.cwdir
        readline.set_completer(None)
        path = raw_input("path or wildcard expr in rule (empty to quit): ")
        path = path.strip()
        if path == '':
            return True
        default = Status.text_to_status('good')
        self.cmpl.set_choices_completion(Status.STATUSES + ['Q'], default)
        while True:
            statuses_text = Status.get_statuses_prompt(", ")
            status = raw_input(statuses_text + " Q(quit)) [%s]: " %
                               default)
            status = status.strip()
            if status == "":
                status = default
            if status[0].upper() in Status.STATUSES:
                status = status[0].upper()
                break
            elif status == 'q' or status == 'Q':
                return None
            else:
                print "Unknown status type"
                continue

        # fixme should check that any wildcard is only one and only
        # in the last component... someday

        if path[0] != os.path.sep:
            path = os.path.join(self.cenv.cwdir, path)
        if path[-1] == os.path.sep:
            path = path[:-1]
            filetype = clouseau.retention.utils.ruleutils.text_to_entrytype('dir')
        else:
            filetype = clouseau.retention.utils.ruleutils.text_to_entrytype('file')

        clouseau.retention.utils.ruleutils.do_add_rule(
            self.cdb, path, filetype, status, self.cenv.host)
        # update the ignores list since we have a new rule
        results = clouseau.retention.utils.ignores.get_ignored_from_rulestore(
            self.cdb, [self.cenv.host])
        if self.cenv.host in results:
            self.ignored_from_rulestore[self.cenv.host] = results[self.cenv.host]
        return True
def row_to_rule(row):
    # ('/home/ariel/wmf/security', '/home/ariel/wmf/security/openjdk6', 'D', 'G')
    (basedir, name, entrytype, status) = row
    basedir = clouseau.retention.utils.rule.from_unicode(basedir)
    name = clouseau.retention.utils.rule.from_unicode(name)
    rule = {'path': os.path.join(basedir, name),
            'type': entrytype_to_text(entrytype),
            'status': Status.status_to_text(status)}
    return rule
def row_to_rule(row):
    # ('/home/ariel/wmf/security', '/home/ariel/wmf/security/openjdk6', 'D', 'G')
    (basedir, name, entrytype, status) = row
    basedir = clouseau.retention.utils.rule.from_unicode(basedir)
    name = clouseau.retention.utils.rule.from_unicode(name)
    rule = {
        'path': os.path.join(basedir, name),
        'type': entrytype_to_text(entrytype),
        'status': Status.status_to_text(status)
    }
    return rule
 def set_up_and_export_rule_store(self):
     hosts = self.cdb.store_db_list_all_hosts()
     destdir = os.path.join(os.path.dirname(self.store_filepath),
                            "data_retention.d")
     if not os.path.isdir(destdir):
         os.makedirs(destdir, 0755)
     for host in hosts:
         all_destpath = os.path.join(destdir, host + "_store.yaml")
         clouseau.retention.utils.ruleutils.export_rules(self.cdb, all_destpath, host)
         good_destpath = os.path.join(destdir, host + "_store_good.yaml")
         clouseau.retention.utils.ruleutils.export_rules(self.cdb, good_destpath, host,
                                                         Status.text_to_status('good'))
 def set_up_and_export_rule_store(self):
     hosts = self.cdb.store_db_list_all_hosts()
     destdir = os.path.join(os.path.dirname(self.store_filepath),
                            "data_retention.d")
     if not os.path.isdir(destdir):
         os.makedirs(destdir, 0755)
     for host in hosts:
         all_destpath = os.path.join(destdir, host + "_store.yaml")
         clouseau.retention.utils.ruleutils.export_rules(
             self.cdb, all_destpath, host)
         good_destpath = os.path.join(destdir, host + "_store_good.yaml")
         clouseau.retention.utils.ruleutils.export_rules(
             self.cdb, good_destpath, host, Status.text_to_status('good'))
def check_args(hosts, action, status):
    if hosts is None:
        usage("Mandatory 'hosts' argument not specified")

    if action is None:
        usage("Mandatory 'action' argument not specified")

    if action not in ['show', 'delete', 'add']:
        usage('unknown action %s specified' % action)

    if status is not None:
        status = Status.status_to_text(status)
        if status is None:
            usage('unknown status %s specified' % status)
Beispiel #15
0
def check_args(hosts, action, status):
    if hosts is None:
        usage("Mandatory 'hosts' argument not specified")

    if action is None:
        usage("Mandatory 'action' argument not specified")

    if action not in ['show', 'delete', 'add']:
        usage('unknown action %s specified' % action)

    if status is not None:
        status = Status.status_to_text(status)
        if status is None:
            usage('unknown status %s specified' % status)
Beispiel #16
0
 def show_menu(self, level):
     if level == 'top':
         text = ("S(set status)/E(examine directory)/"
                 "Filter directory listings/"
                 "I(ignore)/R(manage rules)/Q(quit menu)")
         command = self.get_menu_entry(['S', 'E', 'I', 'F', 'R', 'Q'], 'S',
                                       text)
     elif level == 'status':
         text = Status.get_statuses_prompt(", ") + ", Q(quit status menu)"
         command = self.get_menu_entry(Status.STATUSES + ['Q'], text,
                                       Status.text_to_status('good'))
         if command == 'Q' or command == 'q':
             level = 'top'
     elif level == 'examine':
         text = ("D(down a level)/U(up a level)/E(show entries)/"
                 "C(show contents of file)/R(show rules)/"
                 "F(filter directory listings/"
                 "M(mark file(s))/Q(quit examine menu)")
         command = self.get_menu_entry(
             ['D', 'U', 'E', 'F', 'C', 'R', 'M', 'Q'], 'E', text)
         if command == 'Q' or command == 'q':
             level = 'top'
     elif level == 'rule':
         text = ("S(show all rules of type)/D(show rules covering dir)/"
                 "C(show rules covering dir contents)/"
                 "A(add rule to rules store)/"
                 "R(remove rule from rules store/"
                 "E(export rules from store to file)/"
                 "I(import rules from file to store)/Q(quit rule menu)")
         command = self.get_menu_entry(['S', 'C', 'A', 'R', 'E', 'I', 'Q'],
                                       'D', text)
         if command == 'Q' or command == 'q':
             level = 'top'
     else:
         command = None
     return command
def get_ignored_from_rulestore(cdb, hosts):
    ignored = {}
    for host in hosts:
        rules_from_store = clouseau.retention.utils.ruleutils.get_rules(
            cdb, host, Status.text_to_status('good'))

        if rules_from_store is not None:
            ignored[host] = clouseau.retention.utils.ignores.init_ignored()

            for rule in rules_from_store:
                if rule['status'] != 'good':
                    continue
                path = rule['path']
                if rule['type'] == 'dir':
                    if path.endswith('/'):
                        path = path[:-1]
                    ignored[host]['dirs']['/'].append(path)
                else:
                    ignored[host]['files']['/'].append(path)
    return ignored
Beispiel #18
0
def get_ignored_from_rulestore(cdb, hosts):
    ignored = {}
    for host in hosts:
        rules_from_store = clouseau.retention.utils.ruleutils.get_rules(
            cdb, host, Status.text_to_status('good'))

        if rules_from_store is not None:
            ignored[host] = clouseau.retention.utils.ignores.init_ignored()

            for rule in rules_from_store:
                if rule['status'] != 'good':
                    continue
                path = rule['path']
                if rule['type'] == 'dir':
                    if path.endswith('/'):
                        path = path[:-1]
                    ignored[host]['dirs']['/'].append(path)
                else:
                    ignored[host]['files']['/'].append(path)
    return ignored