def filter_rules_by_name_and_tag(
name: str,
tag: Tuple[str],
exclude_tag: Tuple[str],
session: Optional[Session] = None,
no_strings: Optional[bool] = False) -> Tuple[List[Rule], int]:
if not session:
session = get_session()
rules = session.query(Rule)
if no_strings:
rules = rules.options(noload(Rule.strings))
if tag and len(tag) > 0:
rules = rules.select_from(Tag).join(Rule.tags).filter(
Tag.name.in_(tag))
if exclude_tag and len(exclude_tag) > 0:
# Select rules which have one of the excluded tags and make sure the previously selected rules are not in there.
rules = rules.filter(~Rule.id.in_(
session.query(Rule).select_from(Tag).join(Rule.tags).filter(
Tag.name.in_(exclude_tag)).with_entities(Rule.id)))
if name and len(name) > 0:
rules = rules.filter(Rule.name.like(f"%{name}%"))
count = rules.count()
if count == 0:
return [], count
else:
rules = rules.all()
return rules, len(rules)
def export(name: str, tag: str, single: bool, path: str):
c, ec = Console(), Console(stderr=True, style="bold red")
session = get_session()
rules, count = filter_rules_by_name_and_tag(name, tag, session)
if count == 0:
ec.print(f"Found no matching rules.")
exit(-1)
if single and os.path.isdir(path):
ec.print(f"Given path ({path}) is a directory.")
exit(-1)
yb = YaraBuilder()
for rule in rules:
rule.add_to_yarabuilder(yb)
if not single:
with io.open(os.path.join(path, rule.name + ".yar"), "w") as fh:
fh.write(yb.build_rule(rule.name))
if single:
with io.open(path, "w") as fh:
fh.write(yb.build_rules())
c.print(f"Wrote {len(rules)} rules.")
def edit(identifier: Union[int, str]):
c, ec = Console(), Console(file=stderr, style="bold red")
session = get_session()
rule = get_rule_by_identifier(identifier, session)
if len(rule) > 1:
ec.print(f"Found more than one rule.")
exit(-1)
rule = rule[0]
yb = YaraBuilder()
rule.add_to_yarabuilder(yb)
path, _ = write_ruleset_to_tmp_file(yb)
hash = get_md5(path)
open_file(path, f"{rule.name} opened in external editor...")
edit_hash = get_md5(path)
if hash == edit_hash:
c.print(f"No change detected...")
else:
c.print(f"Change detected, updating rule...")
edited_rule = read_rulefile(path)
if not 0 < len(edited_rule) < 2:
ec.print("Edited rule file must contain exactly one yara rule.")
exit(-1)
rule.name = edited_rule[0].get("rule_name", "Unnamed rule")
rule.meta = plyara_object_to_meta(edited_rule[0])
rule.imports = plyara_object_to_imports(edited_rule[0])
rule.strings = plyara_object_to_strings(edited_rule[0])
rule.tags = plyara_object_to_tags(edited_rule[0], session)
rule.condition = plyara_object_to_condition(edited_rule[0])
rule.ruleset = plyara_object_to_ruleset(edited_rule[0], session)
session.commit()
os.remove(path)
def get_ruleset_by_identifier(identifier: Union[str, int],
session: Optional[Session] = None) -> Ruleset:
if not session:
session = get_session()
ruleset = session.query(Ruleset)
if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier):
return ruleset.get(int(identifier))
else:
return ruleset.filter(Ruleset.name.like(identifier)).first()
def get_rule_by_identifier(identifier: Union[str, int],
session: Optional[Session] = None) -> List[Rule]:
if not session:
session = get_session()
rules = session.query(Rule)
if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier):
rules = rules.filter(Rule.id == int(identifier))
else:
rules = rules.filter(Rule.name.like(f"%{identifier}%"))
return rules.all()
def scan(tag: Tuple[str], exclude_tag: Tuple[str], name: str, timeout: int, no_progress: bool, csv: bool,
recursive: bool, paths: List[str]):
c, ec = Console(), Console(style="bold red", stderr=True)
if len(paths) == 0:
with click.Context(scan) as ctx:
c.print(scan.get_help(ctx))
exit(-1)
session = get_session()
rules, count = filter_rules_by_name_and_tag(name, tag, exclude_tag, session)
if count == 0:
ec.print("No rules matching your criteria.")
exit(-1)
yb = YaraBuilder()
for rule in rules:
rule.add_to_yarabuilder(yb)
ruleset_path, _ = write_ruleset_to_tmp_file(yb)
# Initialize external parameter filename as empty string. This will be filled during matching files.
ruleset_compiled = yara.compile(ruleset_path, externals={
"filename": ""
})
if not no_progress:
c.print(f"Using ruleset {ruleset_path} for scanning. Check the rule file in case any error shows up.")
if recursive:
list_of_files = []
for path in paths:
list_of_files.extend(get_dir_recursive(path))
paths = list_of_files
with Progress() if not no_progress else c as prog:
if isinstance(prog, Progress):
t1 = prog.add_task("Scanning...", total=len(paths))
for path in paths:
if isinstance(prog, Progress):
prog.update(t1, advance=1, description=f"Scanning {path}...")
if os.path.isdir(path):
continue
try:
# Check if file matches. This also adds the basename as filename parameter.
matches = ruleset_compiled.match(path, timeout=timeout, externals={
"filename": os.path.basename(path)
})
except yara.TimeoutError:
prog.print(Text("Timed out!", style="bold red"))
if isinstance(prog, Progress):
prog.update(t1, description="Timed out!")
exit(-1)
for match in matches:
if csv:
prog.print(f'"{match.rule}","{",".join(match.tags)}","{path}"')
else:
prog.print(f"{match.rule} ({', '.join(match.tags)}): {path}", highlight=not no_progress)
if isinstance(prog, Progress):
prog.update(t1, description="Finished scanning!")
os.remove(ruleset_path)
def new():
c = Console()
session = get_session()
path = open_temp_file_with_template()
plyara_list = read_rulefile(path)
for plyara_object in plyara_list:
rule = plyara_obj_to_rule(plyara_object, session)
session.add(rule)
c.print(f"Rule {rule.name} added to database.")
session.commit()
remove(path)
def read():
c = Console()
session = get_session()
stdin = ""
for line in sys.stdin:
stdin += line
plyara_list = parse_rule(stdin)
for plyara_obj in plyara_list:
rule = plyara_obj_to_rule(plyara_obj, session)
session.add(rule)
c.print(f"Added rule {rule.name} from stdin.")
session.commit()
def create(name: str):
c, ec = Console(), Console(stderr=True, style="bold red")
session = get_session()
ruleset = session.query(Ruleset).filter(Ruleset.name == name).first()
if ruleset:
ec.print("Ruleset with that name already exists.")
exit(-1)
ruleset = Ruleset(name=name)
session.add(ruleset)
session.commit()
c.print("New ruleset added.")
def rule(raw: bool, search_term: str):
c = Console()
session = get_session()
rules = session.query(Rule).select_from(Meta).join(Rule.meta).filter(
or_(
Rule.name.like(f"%{search_term}%"),
and_(Meta.key.like("description"),
Meta.value.like(f"%{search_term}%")))).all()
if not raw:
table = rules_to_table(rules)
c.print(table)
else:
c.print(rules_to_highlighted_string(rules))
def ruleset_list():
c = Console()
session = get_session()
rulesets = session.query(Ruleset).all()
if len(rulesets) == 0:
c.print("No rulesets found.")
exit(-1)
t = Table()
t.add_column("ID")
t.add_column("Name")
t.add_column("Number of rules")
for ruleset in rulesets:
t.add_row(str(ruleset.id), ruleset.name, str(len(ruleset.rules)))
c.print(t)
def plyara_object_to_tags(obj: Dict,
session: Optional[Session] = None) -> List[Tag]:
"""Returns a list of initialized Tag objects based on a plyara dict"""
tags: List[Tag] = []
if not session:
session = get_session()
for tag in obj.get("tags", []):
t = session.query(Tag).filter(Tag.name == tag).first()
if t:
tags.append(t)
else:
t = Tag(name=tag)
tags.append(t)
return tags
def get(raw: bool, identifier: str):
c = Console()
session = get_session()
ruleset = get_ruleset_by_identifier(identifier, session)
if not ruleset:
c.print("Ruleset not found.")
exit(-1)
if not raw:
c.print(rules_to_table(ruleset.rules))
else:
yb = YaraBuilder()
for rule in ruleset.rules:
rule.add_to_yarabuilder(yb)
c.print(yb.build_rules())
def delete(identifier: Union[int, str]):
session = get_session()
rule = session.query(Rule)
if isinstance(identifier, int) or re.fullmatch(r"^\d+$", identifier):
rule = rule.filter(Rule.id == int(identifier))
else:
rule = rule.filter(Rule.name.like(f"%{identifier}%"))
rule = rule.all()
rule_names = ", ".join([r.name for r in rule])
confirmed = Confirm.ask(
f"Do you really want to delete the following rules: {rule_names}")
if confirmed:
for r in rule:
session.delete(r)
session.commit()
def scan(tag: str, name: str, timeout: int, no_progress: bool, csv: bool,
file: List[str]):
c, ec = Console(), Console(stderr=True)
if len(file) == 0:
with click.Context(scan) as ctx:
c.print(scan.get_help(ctx))
exit(-1)
session = get_session()
rules, count = filter_rules_by_name_and_tag(name, tag, session)
if count == 0:
ec.print("No rules matching your criteria.")
exit(-1)
yb = YaraBuilder()
for rule in rules:
rule.add_to_yarabuilder(yb)
ruleset_path, _ = write_ruleset_to_tmp_file(yb)
ruleset_compiled = yara.compile(ruleset_path)
if not no_progress:
c.print(
f"Using ruleset {ruleset_path} for scanning. Check the rule file in case any error shows up."
)
with Progress() if not no_progress else c as prog:
if isinstance(prog, Progress):
t1 = prog.add_task("Scanning...", total=len(file))
for path in file:
if isinstance(prog, Progress):
prog.update(t1, advance=1, description=f"Scanning {path}...")
if os.path.isdir(path):
continue
try:
matches = ruleset_compiled.match(path, timeout=timeout)
except yara.TimeoutError:
prog.print(Text("Timed out!", style="bold red"))
if isinstance(prog, Progress):
prog.update(t1, description="Timed out!")
exit(-1)
for match in matches:
if csv:
prog.print(
f'"{match.rule}","{",".join(match.tags)}","{path}"')
else:
prog.print(
f"{match.rule} ({', '.join(match.tags)}): {path}",
highlight=not no_progress)
if isinstance(prog, Progress):
prog.update(t1, description="Finished scanning!")
os.remove(ruleset_path)
def stats():
c = Console()
config = load_config()
db = config.get_current_db()
session = get_session()
rule_count = session.query(Rule).count()
string_count = session.query(String).count()
meta_count = session.query(Meta).count()
tag_count = session.query(Tag).count()
c.print(f"Number of rules:\t{rule_count}")
c.print(f"Number of strings:\t{string_count}")
c.print(f"Number of meta fields:\t{meta_count}")
c.print(f"Number of tags:\t\t{tag_count}")
c.print()
if db["driver"] == "sqlite":
c.print(
f"Database size: \t\t{os.path.getsize(db['path'])/1024/1024:.2}MB")
def filter_rules_by_name_and_tag(
name: str,
tag: str,
session: Optional[Session] = None) -> Tuple[List[Rule], int]:
if not session:
session = get_session()
rules = session.query(Rule)
if tag and len(tag) > 0:
rules = rules.select_from(Tag).join(Rule.tags).filter(Tag.name == tag)
if name and len(name) > 0:
rules = rules.filter(Rule.name.like(f"%{name}%"))
count = rules.count()
if count == 0:
return [], count
else:
return rules.all(), count
def tags(reverse, limit):
c, ec = Console(), Console(stderr=True, style="bold red")
session = get_session()
tags = session.query(Tag).all()
if len(tags) == 0:
ec.print("No tags available.")
exit(-1)
sorted_tags = []
for tag in tags:
sorted_tags.append((tag.name, len(tag.rules)))
sorted_tags.sort(key=lambda x: x[1], reverse=(not reverse))
table = Table()
table.add_column("Tag")
table.add_column("Rule count")
for tag in sorted_tags[:limit]:
table.add_row(tag[0], str(tag[1]))
c.print(table)
def add(paths: List[str]):
session = get_session()
with Progress() as progress:
t1 = progress.add_task("Processing rule files...", total=len(paths))
for rule_path in paths:
progress.console.print(
f"Processing {os.path.basename(rule_path)}...")
plyara_list = parse_rule_file(rule_path)
for plyara_obj in plyara_list:
r = plyara_obj_to_rule(plyara_obj, session)
available_rules_count = session.query(Rule).filter(
Rule.name == r.name).count()
if available_rules_count > 1:
progress.console.print(
f"Rule {r.name} already {available_rules_count - 1} time(s) in the db. "
f"You should rename the new rule!",
style="bold red")
session.add(r)
progress.update(t1, advance=1)
session.commit()
def list(tag: str, raw: bool, name: str, ensure: bool, assign: str):
c, ec = Console(), Console(stderr=True, style="bold yellow")
session = get_session()
rules, count = filter_rules_by_name_and_tag(name, tag, session)
if count == 0:
c.print(f"Query returned empty list of rules.")
exit(-1)
if assign and len(assign) > 0:
ruleset = get_ruleset_by_identifier(assign, session)
if not ruleset:
ec.print("Ruleset not found.")
exit(-1)
for rule in rules:
rule.ruleset = ruleset
session.commit()
if raw:
c.print(rules_to_highlighted_string(rules))
else:
c.print(rules_to_table(rules, ensure=ensure))
def export(name: str, tag: Tuple[str], exclude_tag: Tuple[str], single: bool,
compiled: bool, path: str):
c, ec = Console(), Console(stderr=True, style="bold red")
session = get_session()
rules, count = filter_rules_by_name_and_tag(name, tag, exclude_tag,
session)
path = Path(path)
if count == 0:
ec.print(f"Found no matching rules.")
exit(-1)
if single and path.is_dir():
ec.print(f"Given path ({path}) is a directory.")
exit(-1)
yb = YaraBuilder()
for rule in rules:
rule.add_to_yarabuilder(yb)
yb.write_rules_to_file(path, single_file=single, compiled=compiled)
c.print(f"Wrote {len(rules)} rules.")
def string(raw, query_string):
c = Console()
session = get_session()
strings = session.query(String).filter(String.type == "text").filter(
String.value.like(query_string)).all()
if not raw:
c.print(f"Found {len(strings)} strings.")
t = Table()
t.add_column("String")
t.add_column("ID")
t.add_column("Rule")
t.add_column("Tags")
for string in strings:
t.add_row(string.value, str(string.rule.id), string.rule.name,
", ".join([tag.name for tag in string.rule.tags]))
c.print(t)
else:
yb = YaraBuilder()
for string in strings:
if string.rule.name not in yb.yara_rules.keys():
string.rule.add_to_yarabuilder(yb)
syntax = Syntax(yb.build_rules(), "python", background_color="default")
c.print(syntax)
def export(identifier: str, single: bool, compiled: bool, path: str):
c = Console()
session = get_session()
ruleset = get_ruleset_by_identifier(identifier, session)
if not ruleset:
c.print("Ruleset not found.")
exit(-1)
path = Path(path)
if len(ruleset.rules) == 0:
c.print(f"Found no matching rules.")
exit(-1)
yb = YaraBuilder()
for rule in ruleset.rules:
try:
rule.add_to_yarabuilder(yb)
except ValueError as e:
ruleset.rules.remove(rule)
yb.yara_rules.popitem()
c.print(f"Error:{rule.name} not exported: {e}")
yb.write_rules_to_file(path, single_file=single, compiled=compiled)
c.print(f"Wrote {len(ruleset.rules)} rules.")
