def filter_rules_by_name_and_tag( name: str, tag: Tuple[str], exclude_tag: Tuple[str], session: Optional[Session] = None, no_strings: Optional[bool] = False) -> Tuple[List[Rule], int]: if not session: session = get_session() rules = session.query(Rule) if no_strings: rules = rules.options(noload(Rule.strings)) if tag and len(tag) > 0: rules = rules.select_from(Tag).join(Rule.tags).filter( Tag.name.in_(tag)) if exclude_tag and len(exclude_tag) > 0: # Select rules which have one of the excluded tags and make sure the previously selected rules are not in there. rules = rules.filter(~Rule.id.in_( session.query(Rule).select_from(Tag).join(Rule.tags).filter( Tag.name.in_(exclude_tag)).with_entities(Rule.id))) if name and len(name) > 0: rules = rules.filter(Rule.name.like(f"%{name}%")) count = rules.count() if count == 0: return [], count else: rules = rules.all() return rules, len(rules)
def export(name: str, tag: str, single: bool, path: str): c, ec = Console(), Console(stderr=True, style="bold red") session = get_session() rules, count = filter_rules_by_name_and_tag(name, tag, session) if count == 0: ec.print(f"Found no matching rules.") exit(-1) if single and os.path.isdir(path): ec.print(f"Given path ({path}) is a directory.") exit(-1) yb = YaraBuilder() for rule in rules: rule.add_to_yarabuilder(yb) if not single: with io.open(os.path.join(path, rule.name + ".yar"), "w") as fh: fh.write(yb.build_rule(rule.name)) if single: with io.open(path, "w") as fh: fh.write(yb.build_rules()) c.print(f"Wrote {len(rules)} rules.")
def edit(identifier: Union[int, str]): c, ec = Console(), Console(file=stderr, style="bold red") session = get_session() rule = get_rule_by_identifier(identifier, session) if len(rule) > 1: ec.print(f"Found more than one rule.") exit(-1) rule = rule[0] yb = YaraBuilder() rule.add_to_yarabuilder(yb) path, _ = write_ruleset_to_tmp_file(yb) hash = get_md5(path) open_file(path, f"{rule.name} opened in external editor...") edit_hash = get_md5(path) if hash == edit_hash: c.print(f"No change detected...") else: c.print(f"Change detected, updating rule...") edited_rule = read_rulefile(path) if not 0 < len(edited_rule) < 2: ec.print("Edited rule file must contain exactly one yara rule.") exit(-1) rule.name = edited_rule[0].get("rule_name", "Unnamed rule") rule.meta = plyara_object_to_meta(edited_rule[0]) rule.imports = plyara_object_to_imports(edited_rule[0]) rule.strings = plyara_object_to_strings(edited_rule[0]) rule.tags = plyara_object_to_tags(edited_rule[0], session) rule.condition = plyara_object_to_condition(edited_rule[0]) rule.ruleset = plyara_object_to_ruleset(edited_rule[0], session) session.commit() os.remove(path)
def get_ruleset_by_identifier(identifier: Union[str, int], session: Optional[Session] = None) -> Ruleset: if not session: session = get_session() ruleset = session.query(Ruleset) if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier): return ruleset.get(int(identifier)) else: return ruleset.filter(Ruleset.name.like(identifier)).first()
def get_rule_by_identifier(identifier: Union[str, int], session: Optional[Session] = None) -> List[Rule]: if not session: session = get_session() rules = session.query(Rule) if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier): rules = rules.filter(Rule.id == int(identifier)) else: rules = rules.filter(Rule.name.like(f"%{identifier}%")) return rules.all()
def scan(tag: Tuple[str], exclude_tag: Tuple[str], name: str, timeout: int, no_progress: bool, csv: bool, recursive: bool, paths: List[str]): c, ec = Console(), Console(style="bold red", stderr=True) if len(paths) == 0: with click.Context(scan) as ctx: c.print(scan.get_help(ctx)) exit(-1) session = get_session() rules, count = filter_rules_by_name_and_tag(name, tag, exclude_tag, session) if count == 0: ec.print("No rules matching your criteria.") exit(-1) yb = YaraBuilder() for rule in rules: rule.add_to_yarabuilder(yb) ruleset_path, _ = write_ruleset_to_tmp_file(yb) # Initialize external parameter filename as empty string. This will be filled during matching files. ruleset_compiled = yara.compile(ruleset_path, externals={ "filename": "" }) if not no_progress: c.print(f"Using ruleset {ruleset_path} for scanning. Check the rule file in case any error shows up.") if recursive: list_of_files = [] for path in paths: list_of_files.extend(get_dir_recursive(path)) paths = list_of_files with Progress() if not no_progress else c as prog: if isinstance(prog, Progress): t1 = prog.add_task("Scanning...", total=len(paths)) for path in paths: if isinstance(prog, Progress): prog.update(t1, advance=1, description=f"Scanning {path}...") if os.path.isdir(path): continue try: # Check if file matches. This also adds the basename as filename parameter. matches = ruleset_compiled.match(path, timeout=timeout, externals={ "filename": os.path.basename(path) }) except yara.TimeoutError: prog.print(Text("Timed out!", style="bold red")) if isinstance(prog, Progress): prog.update(t1, description="Timed out!") exit(-1) for match in matches: if csv: prog.print(f'"{match.rule}","{",".join(match.tags)}","{path}"') else: prog.print(f"{match.rule} ({', '.join(match.tags)}): {path}", highlight=not no_progress) if isinstance(prog, Progress): prog.update(t1, description="Finished scanning!") os.remove(ruleset_path)
def new(): c = Console() session = get_session() path = open_temp_file_with_template() plyara_list = read_rulefile(path) for plyara_object in plyara_list: rule = plyara_obj_to_rule(plyara_object, session) session.add(rule) c.print(f"Rule {rule.name} added to database.") session.commit() remove(path)
def read(): c = Console() session = get_session() stdin = "" for line in sys.stdin: stdin += line plyara_list = parse_rule(stdin) for plyara_obj in plyara_list: rule = plyara_obj_to_rule(plyara_obj, session) session.add(rule) c.print(f"Added rule {rule.name} from stdin.") session.commit()
def create(name: str): c, ec = Console(), Console(stderr=True, style="bold red") session = get_session() ruleset = session.query(Ruleset).filter(Ruleset.name == name).first() if ruleset: ec.print("Ruleset with that name already exists.") exit(-1) ruleset = Ruleset(name=name) session.add(ruleset) session.commit() c.print("New ruleset added.")
def rule(raw: bool, search_term: str): c = Console() session = get_session() rules = session.query(Rule).select_from(Meta).join(Rule.meta).filter( or_( Rule.name.like(f"%{search_term}%"), and_(Meta.key.like("description"), Meta.value.like(f"%{search_term}%")))).all() if not raw: table = rules_to_table(rules) c.print(table) else: c.print(rules_to_highlighted_string(rules))
def ruleset_list(): c = Console() session = get_session() rulesets = session.query(Ruleset).all() if len(rulesets) == 0: c.print("No rulesets found.") exit(-1) t = Table() t.add_column("ID") t.add_column("Name") t.add_column("Number of rules") for ruleset in rulesets: t.add_row(str(ruleset.id), ruleset.name, str(len(ruleset.rules))) c.print(t)
def plyara_object_to_tags(obj: Dict, session: Optional[Session] = None) -> List[Tag]: """Returns a list of initialized Tag objects based on a plyara dict""" tags: List[Tag] = [] if not session: session = get_session() for tag in obj.get("tags", []): t = session.query(Tag).filter(Tag.name == tag).first() if t: tags.append(t) else: t = Tag(name=tag) tags.append(t) return tags
def get(raw: bool, identifier: str): c = Console() session = get_session() ruleset = get_ruleset_by_identifier(identifier, session) if not ruleset: c.print("Ruleset not found.") exit(-1) if not raw: c.print(rules_to_table(ruleset.rules)) else: yb = YaraBuilder() for rule in ruleset.rules: rule.add_to_yarabuilder(yb) c.print(yb.build_rules())
def delete(identifier: Union[int, str]): session = get_session() rule = session.query(Rule) if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier): rule = rule.filter(Rule.id == int(identifier)) else: rule = rule.filter(Rule.name.like(f"%{identifier}%")) rule = rule.all() rule_names = ", ".join([r.name for r in rule]) confirmed = Confirm.ask( f"Do you really want to delete the following rules: {rule_names}") if confirmed: for r in rule: session.delete(r) session.commit()
def scan(tag: str, name: str, timeout: int, no_progress: bool, csv: bool, file: List[str]): c, ec = Console(), Console(stderr=True) if len(file) == 0: with click.Context(scan) as ctx: c.print(scan.get_help(ctx)) exit(-1) session = get_session() rules, count = filter_rules_by_name_and_tag(name, tag, session) if count == 0: ec.print("No rules matching your criteria.") exit(-1) yb = YaraBuilder() for rule in rules: rule.add_to_yarabuilder(yb) ruleset_path, _ = write_ruleset_to_tmp_file(yb) ruleset_compiled = yara.compile(ruleset_path) if not no_progress: c.print( f"Using ruleset {ruleset_path} for scanning. Check the rule file in case any error shows up." ) with Progress() if not no_progress else c as prog: if isinstance(prog, Progress): t1 = prog.add_task("Scanning...", total=len(file)) for path in file: if isinstance(prog, Progress): prog.update(t1, advance=1, description=f"Scanning {path}...") if os.path.isdir(path): continue try: matches = ruleset_compiled.match(path, timeout=timeout) except yara.TimeoutError: prog.print(Text("Timed out!", style="bold red")) if isinstance(prog, Progress): prog.update(t1, description="Timed out!") exit(-1) for match in matches: if csv: prog.print( f'"{match.rule}","{",".join(match.tags)}","{path}"') else: prog.print( f"{match.rule} ({', '.join(match.tags)}): {path}", highlight=not no_progress) if isinstance(prog, Progress): prog.update(t1, description="Finished scanning!") os.remove(ruleset_path)
def stats(): c = Console() config = load_config() db = config.get_current_db() session = get_session() rule_count = session.query(Rule).count() string_count = session.query(String).count() meta_count = session.query(Meta).count() tag_count = session.query(Tag).count() c.print(f"Number of rules:\t{rule_count}") c.print(f"Number of strings:\t{string_count}") c.print(f"Number of meta fields:\t{meta_count}") c.print(f"Number of tags:\t\t{tag_count}") c.print() if db["driver"] == "sqlite": c.print( f"Database size: \t\t{os.path.getsize(db['path'])/1024/1024:.2}MB")
def filter_rules_by_name_and_tag( name: str, tag: str, session: Optional[Session] = None) -> Tuple[List[Rule], int]: if not session: session = get_session() rules = session.query(Rule) if tag and len(tag) > 0: rules = rules.select_from(Tag).join(Rule.tags).filter(Tag.name == tag) if name and len(name) > 0: rules = rules.filter(Rule.name.like(f"%{name}%")) count = rules.count() if count == 0: return [], count else: return rules.all(), count
def tags(reverse, limit): c, ec = Console(), Console(stderr=True, style="bold red") session = get_session() tags = session.query(Tag).all() if len(tags) == 0: ec.print("No tags available.") exit(-1) sorted_tags = [] for tag in tags: sorted_tags.append((tag.name, len(tag.rules))) sorted_tags.sort(key=lambda x: x[1], reverse=(not reverse)) table = Table() table.add_column("Tag") table.add_column("Rule count") for tag in sorted_tags[:limit]: table.add_row(tag[0], str(tag[1])) c.print(table)
def add(paths: List[str]): session = get_session() with Progress() as progress: t1 = progress.add_task("Processing rule files...", total=len(paths)) for rule_path in paths: progress.console.print( f"Processing {os.path.basename(rule_path)}...") plyara_list = parse_rule_file(rule_path) for plyara_obj in plyara_list: r = plyara_obj_to_rule(plyara_obj, session) available_rules_count = session.query(Rule).filter( Rule.name == r.name).count() if available_rules_count > 1: progress.console.print( f"Rule {r.name} already {available_rules_count - 1} time(s) in the db. " f"You should rename the new rule!", style="bold red") session.add(r) progress.update(t1, advance=1) session.commit()
def list(tag: str, raw: bool, name: str, ensure: bool, assign: str): c, ec = Console(), Console(stderr=True, style="bold yellow") session = get_session() rules, count = filter_rules_by_name_and_tag(name, tag, session) if count == 0: c.print(f"Query returned empty list of rules.") exit(-1) if assign and len(assign) > 0: ruleset = get_ruleset_by_identifier(assign, session) if not ruleset: ec.print("Ruleset not found.") exit(-1) for rule in rules: rule.ruleset = ruleset session.commit() if raw: c.print(rules_to_highlighted_string(rules)) else: c.print(rules_to_table(rules, ensure=ensure))
def export(name: str, tag: Tuple[str], exclude_tag: Tuple[str], single: bool, compiled: bool, path: str): c, ec = Console(), Console(stderr=True, style="bold red") session = get_session() rules, count = filter_rules_by_name_and_tag(name, tag, exclude_tag, session) path = Path(path) if count == 0: ec.print(f"Found no matching rules.") exit(-1) if single and path.is_dir(): ec.print(f"Given path ({path}) is a directory.") exit(-1) yb = YaraBuilder() for rule in rules: rule.add_to_yarabuilder(yb) yb.write_rules_to_file(path, single_file=single, compiled=compiled) c.print(f"Wrote {len(rules)} rules.")
def string(raw, query_string): c = Console() session = get_session() strings = session.query(String).filter(String.type == "text").filter( String.value.like(query_string)).all() if not raw: c.print(f"Found {len(strings)} strings.") t = Table() t.add_column("String") t.add_column("ID") t.add_column("Rule") t.add_column("Tags") for string in strings: t.add_row(string.value, str(string.rule.id), string.rule.name, ", ".join([tag.name for tag in string.rule.tags])) c.print(t) else: yb = YaraBuilder() for string in strings: if string.rule.name not in yb.yara_rules.keys(): string.rule.add_to_yarabuilder(yb) syntax = Syntax(yb.build_rules(), "python", background_color="default") c.print(syntax)
def export(identifier: str, single: bool, compiled: bool, path: str): c = Console() session = get_session() ruleset = get_ruleset_by_identifier(identifier, session) if not ruleset: c.print("Ruleset not found.") exit(-1) path = Path(path) if len(ruleset.rules) == 0: c.print(f"Found no matching rules.") exit(-1) yb = YaraBuilder() for rule in ruleset.rules: try: rule.add_to_yarabuilder(yb) except ValueError as e: ruleset.rules.remove(rule) yb.yara_rules.popitem() c.print(f"Error:{rule.name} not exported: {e}") yb.write_rules_to_file(path, single_file=single, compiled=compiled) c.print(f"Wrote {len(ruleset.rules)} rules.")