def test_blaklist(self): """build ns rules - test blacklist""" rules = yara.load_rules(blacklist=['hbgary.antide', 'hbgary.fingerprint']) self.assertTrue('hbgary.antidebug' not in rules.namespaces) self.assertTrue('hbgary.fingerprint' not in rules.namespaces) self.assertTrue('hbgary.libs' in rules.namespaces)
def test_build_rules_and_scan(self): """memory - multi-threaded create scan than destroy""" def match_rule(rules, path): for i in range(10): matches = rules.match_path(os.path.join(cdir, sys.executable)) rules.free() cdir = os.path.split(__file__)[0] rules_rootpath = os.path.join(cdir, 'rules') sm = yr_malloc_count() sf = yr_free_count() for i in range(5): #spool up 4 threads rules = yara.load_rules(rules_rootpath, includes=True) target = os.path.join(cdir, sys.executable) tl = [] for i in range(4): t1 = Thread(target=match_rule, args=[rules, target]) t2 = Thread(target=match_rule, args=[rules, target]) t3 = Thread(target=match_rule, args=[rules, target]) t1.start() t2.start() t3.start() tl.append((t1, t2, t3)) for t1, t2, t3 in tl: t1.join() t2.join() t3.join() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def yara_info(self): """ { 'tags': ['foo', 'bar'], 'matches': True, 'namespace': 'default', 'rule': 'my_rule', 'meta': {}, 'strings': [(81L, '$a', 'abc'), (141L, '$b', 'def')] } """ file_path = self.cfg.target_abs_path yara_info = [] if os.path.exists(self.cfg.yara_rules_data): rules = yara.load_rules(self.cfg.yara_rules_data) matches = rules.match(file_path) self.log.info(matches) if len(matches): for item in matches: self.log.info(type(item)) node={} node["ID"] = metrics.S_ID_YARA_INFO node["str"] = item.rule yara_info.append(node) self.log.info(yara_info) self.info["yara_info"] = yara_info
def test_build_rules_and_scan(self): """memory - multi-threaded create scan than destroy""" def match_rule(rules, path): for i in range(10): matches = rules.match_path(sys.executable) rules.free() sm = yr_malloc_count() sf = yr_free_count() for i in range(5): # spool up 4 threads rules = yara.load_rules(RULES_ROOT, blacklist=["broken", "extern"]) target = sys.executable tl = [] for i in range(4): t1 = Thread(target=match_rule, args=[rules, target]) t2 = Thread(target=match_rule, args=[rules, target]) t3 = Thread(target=match_rule, args=[rules, target]) t1.start() t2.start() t3.start() tl.append((t1, t2, t3)) for t1, t2, t3 in tl: t1.join() t2.join() t3.join() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_whitelist_blacklist(self): """build ns rules - test whitelist and blacklist""" rules = yara.load_rules(whitelist=['hbgary'], blacklist=['hbgary.finger', 'hbgary.libs']) self.assertTrue('hbgary.fingerprint' not in rules.namespaces) self.assertTrue('example.packer_rules' not in rules.namespaces) self.assertTrue(len(rules.namespaces) == 6)
def test_build_rules_and_scan(self): """memory - multi-threaded create scan than destroy""" def match_rule(rules, path): for i in range(10): matches = rules.match_path(sys.executable) rules.free() sm = yr_malloc_count() sf = yr_free_count() for i in range(5): #spool up 4 threads rules = yara.load_rules(RULES_ROOT, blacklist=['broken', 'extern']) target = sys.executable tl = [] for i in range(4): t1 = Thread(target=match_rule, args=[rules, target]) t2 = Thread(target=match_rule, args=[rules, target]) t3 = Thread(target=match_rule, args=[rules, target]) t1.start() t2.start() t3.start() tl.append((t1, t2, t3)) for t1, t2, t3 in tl: t1.join() t2.join() t3.join() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_blaklist(self): """build ns rules - test blacklist""" rules = yara.load_rules( blacklist=['hbgary.antide', 'hbgary.fingerprint']) self.assertTrue('hbgary.antidebug' not in rules.namespaces) self.assertTrue('hbgary.fingerprint' not in rules.namespaces) self.assertTrue('hbgary.libs' in rules.namespaces)
def test_changed_root_alternative_prefix(self): """build ns rules - changed root alternative prefix""" rules_rootpath = os.path.join(yara.YARA_RULES_ROOT, 'hbgary') rules = yara.load_rules(rules_rootpath=rules_rootpath, namespace_prefix='external') result = rules.match_path(self.target) self.assertTrue('external.libs' in result)
def test_whitelist(self): """build ns rules - test whitelist""" rules = yara.load_rules(rules_rootpath=RULES_ROOT, whitelist=['private'], blacklist=['broken', 'extern']) self.assertTrue('private' in rules.namespaces) self.assertTrue(len(rules.namespaces) == 1)
def build_rules(fast_match=False, rules_rootpath=yara.YARA_RULES_ROOT, whitelist=[], blacklist=[], include_path=yara.INCLUDE_PATH, rule_filepath=None, externals={}): try: if rule_filepath is None: return yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist, include_path=include_path, externals=externals, fast_match=fast_match) else: return yara.compile(filepath=rule_filepath, externals=externals, fast_match=fast_match, include_path=include_path) except yara.YaraSyntaxError as err: print("Failed to load rules with the following error(s):\n%s" % \ "\n".join([e for _,_,e in err.errors]), file=sys.stderr) if rule_filepath is None: blacklist = set() for f, _, _ in err.errors: f = os.path.splitext(f[len(rules_rootpath) + 1:])[0] blacklist.add(f.replace(os.path.sep, '.')) print("\nYou could blacklist the erroneous rules using:", file=sys.stderr) print(" --blacklist=%s" % ",".join(blacklist), file=sys.stderr) return None
def build_rules(fast_match=False, rules_rootpath=yara.YARA_RULES_ROOT, whitelist=[], blacklist=[], rule_filepath=None, externals={}): try: if rule_filepath is None: return yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist, includes=True, externals=externals, fast_match=fast_match) else: return yara.compile(filepath=rule_filepath, includes=True, externals=externals, fast_match=fast_match) except yara.YaraSyntaxError as err: print("Failed to load rules with the following error(s):\n%s" % \ "\n".join([e for _,_,e in err.errors]), file=sys.stderr) if rule_filepath is None: blacklist = set() for f, _, _ in err.errors: f = os.path.splitext(f[len(rules_rootpath)+1:])[0] blacklist.add(f.replace(os.path.sep, '.')) print("\nYou could blacklist the erroneous rules using:", file=sys.stderr) print(" --blacklist=%s" % ",".join(blacklist), file=sys.stderr) return None
def test_create_destroy(self): """memory - create and destroy loop""" sm = yr_malloc_count() sf = yr_free_count() for i in range(100): rules = yara.load_rules(RULES_ROOT, blacklist=["broken", "extern"]) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_create_destroy_and_scan(self): """memory - create and destroy for each scan""" sm = yr_malloc_count() sf = yr_free_count() for i in range(10): rules = yara.load_rules(RULES_ROOT, blacklist=["broken", "extern"]) matches = rules.match_path(sys.executable) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_create_destroy(self): """memory - create and destroy loop""" sm = yr_malloc_count() sf = yr_free_count() for i in range(100): rules = yara.load_rules(RULES_ROOT, blacklist=['broken', 'extern']) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_create_destroy_and_scan(self): """memory - create and destroy for each scan""" sm = yr_malloc_count() sf = yr_free_count() for i in range(10): rules = yara.load_rules(RULES_ROOT, blacklist=['broken', 'extern']) matches = rules.match_path(sys.executable) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def get_yara_rules(self): # Try to find the yara rules directory relative to the worker my_dir = os.path.dirname(os.path.realpath(__file__)) yara_rule_path = os.path.join(my_dir, 'yara/rules') if not os.path.exists(yara_rule_path): raise Exception('yara could not find yara rules directory under: %s' % os.getcwd()) # Okay load in all the rules under the yara rule path rules = yara.load_rules(rules_rootpath=yara_rule_path) return rules
def get_rules_from_disk(): ''' Recursively traverse the yara/rules directory for rules ''' # Try to find the yara rules directory relative to the worker my_dir = os.path.dirname(os.path.realpath(__file__)) yara_rule_path = os.path.join(my_dir, 'yara/rules') if not os.path.exists(yara_rule_path): raise RuntimeError('yara could not find yara rules directory under: %s' % my_dir) # Okay load in all the rules under the yara rule path rules = yara.load_rules(rules_rootpath=yara_rule_path, fast_match=True) return rules
def test_build_rules_and_scan(self): """memory - create multi scan than destroy""" sm = yr_malloc_count() sf = yr_free_count() rules = yara.load_rules(RULES_ROOT, blacklist=["broken", "extern"]) for i in range(100): matches = rules.match_path(sys.executable) rules.free() del matches del rules dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_create_destroy(self): """memory - create and destroy loop""" cdir = os.path.split(__file__)[0] rules_rootpath = os.path.join(cdir, 'rules') sm = yr_malloc_count() sf = yr_free_count() for i in range(100): rules = yara.load_rules(rules_rootpath, includes=True) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_build_rules_and_scan(self): """memory - create multi scan than destroy""" sm = yr_malloc_count() sf = yr_free_count() rules = yara.load_rules(RULES_ROOT, blacklist=['broken', 'extern']) for i in range(100): matches = rules.match_path(sys.executable) rules.free() del matches del rules dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def test_create_destroy_and_scan(self): """memory - create and destroy for each scan""" cdir = os.path.split(__file__)[0] rules_rootpath = os.path.join(cdir, 'rules') sm = yr_malloc_count() sf = yr_free_count() for i in range(10): rules = yara.load_rules(rules_rootpath, includes=True) matches = rules.match_path(os.path.join(cdir, sys.executable)) rules.free() dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def __init__(self, pids=[], paths=[], rules_rootpath=yara.YARA_RULES_ROOT, whitelist=[], blacklist=[], thread_pool=DEFAULT_THREAD_POOL): """yara scanner class. kwargs: pids - list of process ids to scan paths - list of paths to scan rules_rootpath - path to the root of the rules directory whitelist - whitelist of rules to use in scanner blacklist - blacklist of rules to not use in scanner thread_pool - number of threads to use in scanner """ self._rules = yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist, includes=True) print(self._rules, file=sys.stderr) self._jq = Queue() self._rq = Queue() self._empty = Event() self.scanned = 0 self.quit = Event() for i in range(thread_pool): t = Thread(target=self._run) t.start() if pids: for pid in pids: self._jq.put((self._rules.match_proc, pid)) else: for path in paths: for p in glob(path): if os.path.isdir(p): for dirpath, dirnames, filenames in os.walk(p): for filename in filenames: a = os.path.join(dirpath, filename) self._jq.put((self._rules.match_path, a)) else: self._jq.put((self._rules.match_path, p)) self._jq.put(None)
def test_build_rules_and_scan(self): """memory - create multi scan than destroy""" cdir = os.path.split(__file__)[0] rules_rootpath = os.path.join(cdir, 'rules') sm = yr_malloc_count() sf = yr_free_count() rules = yara.load_rules(rules_rootpath, includes=True) for i in range(1000): matches = rules.match_path(os.path.join(cdir, sys.executable)) rules.free() del matches del rules dsm = yr_malloc_count() dsf = yr_free_count() self.assertEqual(dsm, dsf)
def __init__(self, pids=[], paths=[], rules_rootpath=yara.YARA_RULES_ROOT, whitelist=[], blacklist=[], thread_pool=DEFAULT_THREAD_POOL): """Scanner yields scan results in a tuple of (path|pid, result) kwargs: pids - list of process ids to scan paths - globbed out list of paths to scan rules_rootpath - path to the root of the rules directory whitelist - whitelist of rules to use in scanner blacklist - blacklist of rules to not use in scanner thread_pool - number of threads to use in scanner """ self._rules = yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist, includes=True) print(self._rules, file=sys.stderr) self._jq = Queue() self._rq = Queue() self._empty = Event() self.scanned = 0 self.quit = Event() for i in range(thread_pool): t = Thread(target=self._run) t.start() if pids: for pid in pids: self._jq.put((self._rules.match_proc, pid)) else: for path in paths: for p in glob(path): if os.path.isdir(p): for dirpath, dirnames, filenames in os.walk(p): for filename in filenames: a = os.path.join(dirpath, filename) self._jq.put((self._rules.match_path, a)) else: self._jq.put((self._rules.match_path, p)) self._jq.put(None)
def test_whitelist(self): """build ns rules - test whitelist""" rules = yara.load_rules(whitelist=['exam', 'hbgary.l']) self.assertTrue('hbgary.libs' in rules.namespaces) self.assertTrue('example.packer_rules' in rules.namespaces) self.assertTrue(len(rules.namespaces) == 2)
def test_good_load(self): """build ns rules - default load""" rules = yara.load_rules(rules_rootpath=RULES_ROOT, blacklist=['broken', 'extern']) result = rules.match_data("dogs dog doggy") self.assertTrue('dogs.meta' in result)
def main(args): try: opts, args = getopt(args, 'hw:b:t:o:', ['proc', 'whitelist=', 'blacklist=', 'thread_pool=', 'root=', 'list', 'fmt=', 'help']) except Exception as exc: print("Getopt error: %s" % (exc), file=sys.stderr) return -1 whitelist = [] blacklist = [] thread_pool = 4 pids = [] paths = args out = sys.stdout out_fmt = str rules_rootpath = yara.YARA_RULES_ROOT list_rules = False for opt, arg in opts: if opt in ['-h', '--help']: print(__help__) return 0 elif opt in ['--root']: if os.path.exists(rules_rootpath): print("root path '%s' does not exist" % arg, file=sys.stderr) return -1 rules_rootpath = arg elif opt in ['--list']: list_rules = True elif opt in ['-o']: out = open(arg, 'wb') elif opt in ['-w', '--whitelist']: whitelist = arg.split(',') elif opt in ['b', '--blacklist']: blacklist = arg.split(',') elif opt in ['--fmt']: if arg == 'pickle': out_fmt = pickle.dumps elif arg == 'json': out_fmt = lambda a: json.dumps(a, ensure_ascii=False, check_circular=False, indent=4) elif arg == 'pprint': out_fmt = pprint.pformat elif arg == 'marshal': out_fmt = marshal.dumps elif arg == 'dict': out_fmt = str else: print("unknown output format %s" % arg, file=sys.stderr) return -1 elif opt in ['t', '--thread_pool']: try: thread_pool = int(arg) except ValueError: print("-t param %s was not an int" % (arg), file=sys.stderr) return -1 elif opt in ['--proc']: paths = [] if not args: print("no PIDs specified") return -1 for pid in args: try: pids.append(int(pid)) except ValueError: print("PID %s was not an int" % (pid), file=sys.stderr) if list_rules is True: rules = yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist) print(rules) return 0 scanner = Scanner(paths=paths, pids=pids, rules_rootpath=rules_rootpath, whitelist=whitelist, blacklist=blacklist, thread_pool=thread_pool) try: status_template = "scan queue: %-7s result queue: %-7s" i = 0 for arg, res in scanner: i += 1 if i % 20 == 0: status = status_template % (scanner.sq_size, scanner.rq_size) sys.stderr.write("\b" * len(status) + status) if res: print("<scan arg='%s'>" % arg, file=out) print(out_fmt(res), file=out) print("</scan>", file=out) finally: scanner.quit.set() status = status_template % (scanner.sq_size, scanner.rq_size) sys.stderr.write("\b" * len(status) + status) print("\nscanned %s items... done." % scanner.scanned, file=sys.stderr)
import sys import json import pika import time from pyinotify import WatchManager, Notifier, ThreadedNotifier, EventsCodes, ProcessEvent import pyinotify MQHOST = sys.argv[1] yaraengine = None try: yaraengine = yara.load_rules(rules_rootpath = "%s/yrules" % os.path.dirname(os.path.realpath(__file__))) except: yaraengine = yara.compile(filepath = "%s/yrules/web.yar" % os.path.dirname(os.path.realpath(__file__))) PROCS = 10 def dofilter(s): if type(s) == list: s = "".join(s) s= s.replace(':','') s=s.decode('hex') return "".join(filter(lambda x: ord(x)<128, s))
def test_default_load(self): """build ns rules - default load""" rules = yara.load_rules() result = rules.match_path(self.target) self.assertTrue('hbgary.libs' in result)
def main(args): try: opts, args = getopt(args, 'hw:b:t:o:', [ 'proc', 'whitelist=', 'blacklist=', 'thread_pool=', 'root=', 'list', 'fmt=', 'help' ]) except Exception as exc: print("Getopt error: %s" % (exc), file=sys.stderr) return -1 whitelist = [] blacklist = [] thread_pool = 4 pids = [] paths = args out = sys.stdout out_fmt = str rules_rootpath = yara.YARA_RULES_ROOT list_rules = False for opt, arg in opts: if opt in ['-h', '--help']: print(__help__) return 0 elif opt in ['--root']: if os.path.exists(rules_rootpath): print("root path '%s' does not exist" % arg, file=sys.stderr) return -1 rules_rootpath = arg elif opt in ['--list']: list_rules = True elif opt in ['-o']: out = open(arg, 'wb') elif opt in ['-w', '--whitelist']: whitelist = arg.split(',') elif opt in ['b', '--blacklist']: blacklist = arg.split(',') elif opt in ['--fmt']: if arg == 'pickle': out_fmt = pickle.dumps elif arg == 'json': out_fmt = lambda a: json.dumps( a, ensure_ascii=False, check_circular=False, indent=4) elif arg == 'pprint': out_fmt = pprint.pformat elif arg == 'marshal': out_fmt = marshal.dumps elif arg == 'dict': out_fmt = str else: print("unknown output format %s" % arg, file=sys.stderr) return -1 elif opt in ['t', '--thread_pool']: try: thread_pool = int(arg) except ValueError: print("-t param %s was not an int" % (arg), file=sys.stderr) return -1 elif opt in ['--proc']: paths = [] if not args: print("no PIDs specified") return -1 for pid in args: try: pids.append(int(pid)) except ValueError: print("PID %s was not an int" % (pid), file=sys.stderr) if list_rules is True: rules = yara.load_rules(rules_rootpath=rules_rootpath, blacklist=blacklist, whitelist=whitelist) print(rules) return 0 scanner = Scanner(paths=paths, pids=pids, rules_rootpath=rules_rootpath, whitelist=whitelist, blacklist=blacklist, thread_pool=thread_pool) try: status_template = "scan queue: %-7s result queue: %-7s" i = 0 for arg, res in scanner: i += 1 if i % 20 == 0: status = status_template % (scanner.sq_size, scanner.rq_size) sys.stderr.write("\b" * len(status) + status) if res: print("<scan arg='%s'>" % arg, file=out) print(out_fmt(res), file=out) print("</scan>", file=out) finally: scanner.quit.set() status = status_template % (scanner.sq_size, scanner.rq_size) sys.stderr.write("\b" * len(status) + status) print("\nscanned %s items... done." % scanner.scanned, file=sys.stderr)
def test_good_load(self): """build ns rules - default load""" rules = yara.load_rules(rules_rootpath=RULES_ROOT, blacklist=["broken", "extern"]) result = rules.match_data("dogs dog doggy") self.assertTrue("dogs.meta" in result)
def test_whitelist(self): """build ns rules - test whitelist""" rules = yara.load_rules(rules_rootpath=RULES_ROOT, whitelist=["private"], blacklist=["broken", "extern"]) self.assertTrue("private" in rules.namespaces) self.assertTrue(len(rules.namespaces) == 1)