def __init__(self, string=None, scan_physical=False, yara_file=None, yara_expression=None, **kwargs): """Scan using yara signatures. Args: string: A verbatim string to search for. we scan their entire address spaces. scan_physical: If true we scan the physical address space. yara_file: The yara file to read. yara_expression: If provided we scan for this yarra expression. """ super(LinYaraScan, self).__init__(**kwargs) if yara_expression: self.rules_source = yara_expression self.rules = yara.compile(source=self.rules_source) elif string: self.rules_source = 'rule r1 {strings: $a = "%s" condition: $a}' % string self.rules = yara.compile(source=self.rules_source) elif yara_file: self.rules = yara.compile(yara_file) else: raise plugin.PluginError("You must specify a yara rule file or " "string to match.") self.scan_physical = scan_physical
def run(self): if os.path.isfile(self.custom_rule): rules = yara.compile(self.custom_rule) elif isinstance(self.custom_rule, str): rules = yara.compile(source=self.custom_rule) matches = [] count = 0 for root, dirs, files in os.walk(self.target): for entry in files: count += 1 pbar = progressbar.ProgressBar(widgets=[progressbar.Bar('+'), ' ', progressbar.Percentage(), ' | ', progressbar.ETA(), ' | ', progressbar.SimpleProgress()], maxval=count).start() p = 0 for root, dirs, files in os.walk(self.target+'\\'): for entry in files: p += 1 pbar.update(p) e = os.path.join(root, entry) try: m = rules.match(e) if len(m) > 1: pass if m: matches.append({'match': m, 'file': e}) except Exception, err: pass
def compile_rules(self): """Compile the YARA rules from command-line parameters. @returns: a YARA object on which you can call 'match' This function causes the plugin to exit if the YARA rules have syntax errors or are not supplied correctly. """ rules = None try: if self._config.YARA_RULES: s = self._config.YARA_RULES # Don't wrap hex or regex rules in quotes if s[0] not in ("{", "/"): s = '"' + s + '"' # Scan for unicode strings if self._config.WIDE: s += "wide" rules = yara.compile(sources = { 'n' : 'rule r1 {strings: $a = ' + s + ' condition: $a}' }) elif self._config.YARA_FILE: rules = yara.compile(self._config.YARA_FILE) elif self._config.YARA_RULES_DIRECTORY: filepaths = {} for filename in glob.glob(self._config.YARA_RULES_DIRECTORY + '/*.rule'): base=os.path.basename(filename) namespace = os.path.splitext(base)[0] filepaths[namespace] = filename rules = yara.compile(filepaths = filepaths) else: debug.error("You must specify a string (-Y) or a rules file (-y)") except yara.SyntaxError, why: debug.error("Cannot compile rules: {0}".format(str(why)))
def YARACompile(ruledata): if ruledata.startswith('#'): if ruledata.startswith('#h#'): rule = binascii.a2b_hex(ruledata[3:]) elif ruledata.startswith('#b#'): rule = binascii.a2b_base64(ruledata[3:]) elif ruledata.startswith('#s#'): rule = 'rule string {strings: $a = "%s" ascii wide nocase condition: $a}' % ruledata[3:] elif ruledata.startswith('#q#'): rule = ruledata[3:].replace("'", '"') elif ruledata.startswith('#x#'): rule = 'rule hexadecimal {strings: $a = { %s } condition: $a}' % ruledata[3:] elif ruledata.startswith('#r#'): rule = 'rule regex {strings: $a = /%s/ ascii wide nocase condition: $a}' % ruledata[3:] else: rule = ruledata[1:] return yara.compile(source=rule, externals={'streamname': '', 'VBA': False}) else: dFilepaths = {} if os.path.isdir(ruledata): for root, dirs, files in os.walk(ruledata): for file in files: filename = os.path.join(root, file) dFilepaths[filename] = filename else: for filename in ProcessAt(ruledata): dFilepaths[filename] = filename return yara.compile(filepaths=dFilepaths, externals={'streamname': '', 'VBA': False})
def _compile_rules(self): """Compile the YARA rules from command-line parameters. @returns: a YARA object on which you can call 'match' This function causes the plugin to exit if the YARA rules have syntax errors or are not supplied correctly. """ rules = None try: if self._config.YARA_RULES: s = self._config.YARA_RULES # Don't wrap hex or regex rules in quotes if s[0] not in ("{", "/"): s = '"' + s + '"' # Option for case insensitive searches if self._config.CASE: s += " nocase" # Scan for unicode and ascii strings if self._config.WIDE: s += " wide ascii" rules = yara.compile(sources = { 'n' : 'rule r1 {strings: $a = ' + s + ' condition: $a}' }) elif self._config.YARA_FILE and os.path.isfile(self._config.YARA_FILE): rules = yara.compile(self._config.YARA_FILE) else: debug.error("You must specify a string (-Y) or a rules file (-y)") except yara.SyntaxError, why: debug.error("Cannot compile rules: {0}".format(str(why)))
def init_yara_rules(self): ''' This method tries to find and compile the yara rules specified by 'family' before the q&a test starts. ''' if (self.method == 'STATIC' or self.method == 'ALL'): if not os.path.isfile(self.YARA_STATIC_DIR+self.family+'.yara'): yara_path = self.YARA_STATIC_DIR+self.family+'.yara' self.die("Can't found static yaras for this family! {0}".format(str(yara_path))) yara_path = self.YARA_STATIC_DIR+self.family+'.yara' rule_static = yara.compile(filepath=yara_path) if not rule_static: self.die("Couldn't compile the .yara! {0}".format(str(yara_path))) if (self.method == 'MEMORY' or self.method == 'ALL'): if not os.path.isfile(self.YARA_MEMORY_DIR+self.family+'.yara'): yara_path = self.YARA_MEMORY_DIR+self.family+'.yara' self.die("Can't found memory yaras for this family! {0}".format(str(yara_path))) yara_path = self.YARA_MEMORY_DIR+self.family+'.yara' rule_memory = yara.compile(filepath=yara_path) if not rule_memory: self.die("Couldn't compile the .yara! {0}".format(str(yara_path))) if (self.method == 'STATIC' or self.method == 'ALL'): return rule_static else: return rule_memory
def testExternals(self): r = yara.compile(source='rule test { condition: ext_int == 15 }', externals={'ext_int': 15}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_bool }', externals={'ext_bool': True}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_bool }', externals={'ext_bool': False}) self.assertFalse(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str contains "ssi" }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /foo/ }', externals={'ext_str': ''}) self.assertFalse(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /ssi(s|p)/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /ppi$/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /ssi$/ }', externals={'ext_str': 'mississippi'}) self.assertFalse(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /^miss/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str matches /ssi$/ }', externals={'ext_str': 'mississippi'}) self.assertFalse(r.match(data='dummy'))
def _yara_check(targ): # Utilize yara rules packed into this script to check for nasty stuff. Compile each rule stored in tmp and check the rule_match results for a Match True hit. contains_pe_file_rule = yara.compile(filepath='/tmp/automal/contains_pe_file.yara') match_pe = contains_pe_file_rule.match(targ) yara_pe_res = "" if len(match_pe) == 1: match_data_pe = str(match_pe).split(',') for item in match_data_pe: if "True" in item.strip() and "matches" in item.strip(): yara_pe_res = "Present_PE_Hit" if yara_pe_res != "Present_PE_Hit": yara_pe_res = "PE_None" else: if yara_pe_res != "Present_PE_Hit": yara_pe_res = "PE_None" maldoc_file_rule = yara.compile(filepath='/tmp/automal/maldoc.yara') match_maldoc = maldoc_file_rule.match(targ) yara_maldoc_res = "" if len(match_maldoc) == 1: match_data_maldoc = str(match_maldoc).split(',') for item in match_data_maldoc: if "True" in item.strip() and "matches" in item.strip(): yara_maldoc_res = "Present_Maldoc_Hit" if yara_maldoc_res != "Present_Maldoc_Hit": yara_maldoc_res = "Maldoc_None" else: if yara_maldoc_res != "Present_Maldoc_Hit": yara_maldoc_res = "Maldoc_None" return(yara_pe_res, yara_maldoc_res)
def calculate(self): """Required: Use Filescan to find Chaperone config file""" if not HAS_YARA: debug.error('Yara must be installed for this plugin') # Complile yara signatures rules_json = yara.compile(sources=YARA_JSON) rules_devices = yara.compile(sources=YARA_HMD) rules_activity = yara.compile(sources=YARA_HMD_ACTIVITY) # Load address space addr_space = utils.load_as(self._config) # Get list of processes tasks = win32.tasks.pslist(addr_space) # Read the Chaperone information from the provided file if self._config.CHAP_FILE: print("Loading Chaperone information from file") file1 = open(self._config.CHAP_FILE, "r+") json_from_file = json.load(file1) self.build_obj(1, json_from_file) for task in tasks: if self._config.FULL_SCAN and str(task.ImageFileName) != "vrmonitor.exe": continue else: print("Scanning {0} pid: {1}".format(task.ImageFileName, task.UniqueProcessId)) vad_offset = 0 for vad, process_space in task.get_vads(): vad_offset += vad.Length if vad.Length > 8*1024*1024*1024: continue # read Vad content data = process_space.zread(vad.Start, vad.Length) if not self._config.CHAP_FILE: # match yara rules for chaperone Json matches = rules_json.match(data=data) self.pull_chaperone(matches, process_space, vad) # Check for tracked device signatures matches = rules_devices.match(data=data) for match in matches: pointer = vad.Start + match.strings[0][0] device_pointer = self.follow_pointers(DEVICE_OFFSETS, pointer, process_space, "HMD") self.get_coords(device_pointer, process_space) # Pull tracked device activity state data matches = rules_activity.match(data=data) for match in matches: pointer = vad.Start + match.strings[0][0] hmd_activity = self.follow_pointers(HMD_ACTIVITY_OFFSETS,pointer,process_space,"Activity") print("HMD activity: {0}".format(activity_dict(hmd_activity & 0xFFFFFFFF))) hmd_state = self.follow_pointers(HMD_STATE_OFFSETS, pointer, process_space, "State") print("HMD state: {0}".format(state_dict(hmd_state & 0xFFFFFFFF)))
def get_yara(self, rulepath=YARA_RULEPATH): """Get Yara signatures matches. @return: matched Yara signatures. """ results = [] if not HAVE_YARA: if not File.notified_yara: File.notified_yara = True log.warning("Unable to import yara (please compile from sources)") return results if not os.path.exists(rulepath): log.warning("The specified rule file at %s doesn't exist, skip", rulepath) return results if not os.path.getsize(self.file_path): return results try: try: filepath = "" filename = "" if self.file_name: filepath = self.file_name filename = self.file_name if self.guest_paths: filepath = self.guest_paths[0] rules = yara.compile(rulepath, externals={"filepath":filepath, "filename":filename}) except: rules = yara.compile(rulepath) matches = rules.match(self.file_path) if getattr(yara, "__version__", None) == "1.7.7": return self._yara_matches_177(matches) results = [] for match in matches: strings = set() for s in match.strings: strings.add(self._yara_encode_string(s[2])) results.append({ "name": match.rule, "meta": match.meta, "strings": list(strings), }) except Exception as e: log.exception("Unable to match Yara signatures: %s", e) return results
def _yara_check(targ): # Utilize yara rules packed into this script to check for nasty stuff. Compile each rule stored in tmp and check the rule_match results for a Match True hit. # -------------------------------------- RULE 1 contains_pe_file_rule = yara.compile(filepath=str(os.path.expanduser("~")) + "/automal/contains_pe_file.yara") match_pe = contains_pe_file_rule.match(targ) yara_pe_res = "" if len(match_pe) == 1: match_data_pe = str(match_pe).split(",") for item in match_data_pe: if "True" in item.strip() and "matches" in item.strip(): yara_pe_res = "Present_PE_Hit" if yara_pe_res != "Present_PE_Hit": yara_pe_res = "PE_None" else: if yara_pe_res != "Present_PE_Hit": yara_pe_res = "PE_None" # -------------------------------------- RULE 2 maldoc_file_rule = yara.compile(filepath=str(os.path.expanduser("~")) + "/automal/maldoc.yara") match_maldoc = maldoc_file_rule.match(targ) yara_maldoc_res = "" if len(match_maldoc) == 1: match_data_maldoc = str(match_maldoc).split(",") for item in match_data_maldoc: if "True" in item.strip() and "matches" in item.strip(): yara_maldoc_res = "Present_Maldoc_Hit" if yara_maldoc_res != "Present_Maldoc_Hit": yara_maldoc_res = "Maldoc_None" else: if yara_maldoc_res != "Present_Maldoc_Hit": yara_maldoc_res = "Maldoc_None" # ------------------------------------- RULE 3 contains_obfus_str = yara.compile(filepath=str(os.path.expanduser("~")) + "/automal/obfus_strings.yara") match_obf = contains_obfus_str.match(targ) yara_obf_res = "" if len(match_obf) == 1: match_data_obfus = str(match_obf).split(",") for item in match_data_obfus: if "True" in item.strip() and "matches" in item.strip(): yara_obf_res = "Present_Obfus_Hit" if yara_obf_res != "Present_Obfus_Hit": yara_obf_res = "Obfus_None" else: if yara_obf_res != "Present_Obfus_Hit": yara_obf_res = "Obfus_None" return (yara_pe_res, yara_maldoc_res, yara_obf_res)
def calculate(self): all_tasks = pstasks.mac_tasks(self._config).allprocs() bit_tasks = [] try: if self._config.PID: # find tasks given PIDs pidlist = [int(p) for p in self._config.PID.split(',')] bit_tasks = [t for t in all_tasks if t.p_pid in pidlist] else: # find multibit process name_re = re.compile("JavaApplicationS", re.I) bit_tasks = [t for t in all_tasks if name_re.search(str(t.p_comm))] except: pass if len(bit_tasks) == 0: yield (None, None) # scan for bitcoin addresses with yara, 34 chars, https://en.bitcoin.it/wiki/Address # Most Bitcoin addresses are 34 characters. They consist of random digits and uppercase # and lowercase letters, with the exception that the uppercase letter "O", uppercase # letter "I", lowercase letter "l", and the number "0" are never used to prevent visual ambiguity. bit_addrs = [] addr_rule = yara.compile(sources = {'n' : 'rule r1 {strings: $a = /[1-9a-zA-z]{34}(?!OIl)/ condition: $a}'}) for task in bit_tasks: scanner = mac_yarascan.MapYaraScanner(task = task, rules = addr_rule) for hit, address in scanner.scan(): content = scanner.address_space.zread(address, 34) if pyenc.is_valid_bitcoin_address(content) and content not in bit_addrs: bit_addrs.append(content) # scan for bitcoin keys with yara, 52 char compressed base58, starts with L or K, https://en.bitcoin.it/wiki/Private_key addr_key = {} key_rule = yara.compile(sources = {'n' : 'rule r1 {strings: $a = /(L|K)[0-9A-Za-z]{51}/ condition: $a}'}) for task in bit_tasks: scanner = mac_yarascan.MapYaraScanner(task = task, rules = key_rule) for hit, address in scanner.scan(): content = scanner.address_space.zread(address, 52) if pyenc.is_valid_wif(content): secret_exp = pyenc.wif_to_secret_exponent(content) key = pykey.Key(secret_exponent = secret_exp,is_compressed=True) if key.address() not in addr_key.keys(): addr_key[key.address()] = content yield(content, key.address()) # addresses with no known keys for bit_addr in bit_addrs: if bit_addr not in addr_key.keys(): yield ("UNKNOWN", bit_addr)
def build_ruleset(): if ruletype == "FILE": try: rules = yara.compile(str(RULES)) print "..... Ruleset Compilation Successful." return rules except: print "[!] - Could not compile YARA rule: %s" % RULES print "Exiting." sys.exit() elif ruletype == "FOLDER": RULEDATA="" # Get list of files ending in .yara RULE_COUNT = len(glob.glob1(RULES,"*.yar")) if RULE_COUNT != 0: for yara_file in glob.glob(os.path.join(RULES, "*.yar")): try: yara.compile(str(yara_file)) print "..... Syntax appears to be OK: %s " % yara_file try: with open(yara_file, "r") as sig_file: file_contents=sig_file.read() RULEDATA=RULEDATA + "\n" + file_contents except: print "..... SKIPPING: Could not open file for reading: %s " % yara_file except: print "..... SKIPPING: Could not compile rule: %s " % yara_file try: rules = yara.compile(source=RULEDATA) print "..... SUCCESS! Compiled noted yara rulesets.\n" return rules except: print "[!] - Some catastropic error occurred in the " \ "compilation of signatureswithin the directory. Exiting." sys.exit() else: print "No files ending in .yar within: %s " % RULES print "Exiting." sys.exit() elif ruletype == "DEFAULT": rules = yara.compile(str(RULES)) print "[+] - Ruleset Compilation Successful." return rules else: print "[!] - ERROR: Possible catastrophic error on build_ruleset. Exiting." sys.exit()
def yara_import_rules(yara_folder): """ Import a folder of YARA rule files Arguments: yara_folder: path to folder containing rules Results: rules: a yara.Rules structure of available YARA rules """ yara_files = {} if not yara_folder[-1] == '\\': yara_folder += '\\' print('[*] Loading YARA rules from folder: %s' % yara_folder) files = os.listdir(yara_folder) for file_name in files: if '.yara' in file_name: yara_files[file_name.split('.yara')[0]] = yara_folder + file_name try: rules = yara.compile(filepaths=yara_files) print('[*] YARA rules loaded. Total files imported: %d' % len(yara_files)) except yara.SyntaxError: print('[!] Syntax error found in one of the imported YARA files. Error shown below.') rules = '' yara_rule_check(yara_folder) print('[!] YARA rules disabled until all Syntax Errors are fixed.') return rules
def check(self, file): """ Checks a given file against all available yara rules :param file: Path to file :type file:str :returns: Python list with matched rules info :rtype: list """ result = [] all_matches = [] for filerules in os.listdir(self.rulepaths): try: rule = yara.compile(os.path.join(self.rulepaths, filerules)) except yara.SyntaxError: continue matches = rule.match(file) if len(matches) > 0: for rulem in matches: rule_family = "_".join([x for x in rulem.rule.replace("_", ".", 1).split("_")[:-1]]) if rule_family not in all_matches: all_matches.append(rule_family) for rule_family in all_matches: rules_info_txt = requests.get('{}/family/{}'.format(self.baseurl, rule_family), auth=HTTPBasicAuth(self.user, self.pwd)) rules_info_json = json.loads(rules_info_txt.text) result.append({ 'family': rule_family, 'common_name': rules_info_json['common_name'], 'description': rules_info_json['description'], 'attribution': rules_info_json['attribution'], 'alt_names': rules_info_json['alt_names'], 'urls': rules_info_json['urls'] }) return result
def compile_rule(self, rule): self.rules_source = rule try: self.rules = yara.compile(source=rule) except Exception as e: raise plugin.PluginError( "Failed to compile yara expression: %s" % e)
def testCompileFiles(self): tmpdir = tempfile.gettempdir() p1 = os.path.join(tmpdir, 'test1') f1 = open(p1, 'wt') f1.write('rule test1 { condition: true }') f1.close() p2 = os.path.join(tmpdir, 'test2') t2 = open(p2, 'wt') t2.write('rule test2 { condition: true }') t2.close() r = yara.compile(filepaths={ 'test1': p1, 'test2': p2 }) self.assertTrue(len(r.match(data='dummy')) == 2) for m in r.match(data='dummy'): self.assertTrue(m.rule in ('test1', 'test2')) self.assertTrue(m.namespace == m.rule) os.remove(p1) os.remove(p2)
def init_rules(self): r = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.yara_rules) if not os.path.exists(r): self.logger.error("Default yara rule file {}: not found".format(r)) return self.rules = yara.compile(r)
def _compile_rules(sigdir, sigfiles): if not sigfiles or not sigdir: raise ServiceConfigError("No signature files specified.") sigsets = [] for sigfile in sigfiles: sigfile = os.path.abspath(os.path.join(sigdir, sigfile.strip())) logger.debug("Full path to file file: %s" % sigfile) filename = os.path.basename(sigfile) dirname = os.path.dirname(sigfile) old = os.getcwd() try: with open(sigfile, "rt") as f: data = f.read() os.chdir(dirname) except Exception as e: logger.exception("File cannot be opened: %s" % sigfile) raise ServiceConfigError(str(e)) try: rules = yara.compile(source=data) except yara.SyntaxError as e: message = "Yara rules file: %s: %s" % (sigfile, str(e)) logger.exception(message) os.chdir(old) raise ServiceConfigError(message) sigsets.append({'name': filename, 'rules': rules}) os.chdir(old) logger.debug(str(sigsets)) return sigsets
def SetRules(self, rules_string): """Sets the rules that the Yara analyzer will use. Args: rules_string(str): Yara rule definitions """ self._rules = yara.compile(source=rules_string)
def yara_scan(raw_data): yara_rules = yara.compile(rule_file) matches = yara_rules.match(data=raw_data) if len(matches) > 0: return str(matches[0]) else: return
def main(): """ Main logic for program """ cfg = configparser.ConfigParser() cfg.read('service.conf') rule_location = 'rules.yar' get_remote = False # Parse configuration options if cfg.has_section('yara_rules'): rule_location = cfg['yara_rules'].get('local_path', fallback='rules.yar') get_remote = cfg['yara_rules'].getboolean('get_remote', fallback=False) if get_remote: DownloadFile(cfg['yara_rules'].get('download_url'), rule_location) else: print("Configuration Error: Cannot find yara_rules section in" "service.conf. Using default values.") # attempt to compile rules try: rules = yara.compile(rule_location) rules.save(rule_location) except yara.YaraSyntaxError: print("Syntax error in the YARA rules.") sys.exit(ERROR_CODE) except yara.YaraError: print("Unknown YARA error.") sys.exit(ERROR_CODE) except Exception as e: print(e) sys.exit(ERROR_CODE) return 0
def get_yara(self, rulepath=os.path.join(CUCKOO_ROOT, "data", "yara", "index.yar")): """Get Yara signatures matches. @return: matched Yara signatures. """ matches = [] if HAVE_YARA: try: rules = yara.compile(rulepath) for match in rules.match(self.file_path): strings = [] for s in match.strings: # Beware, spaghetti code ahead. try: new = s[2].encode("utf-8") except UnicodeDecodeError: s = s[2].lstrip("uU").encode("hex").upper() s = " ".join(s[i:i+2] for i in range(0, len(s), 2)) new = "{ %s }" % s if new not in strings: strings.append(new) matches.append({"name" : match.rule, "meta" : match.meta, "strings" : strings}) except yara.Error as e: log.warning("Unable to match Yara signatures: %s", e) return matches
def __init__(self, outputs, logger, **config): """Creates an instance of the YARA binding. This binding integrates with YARA tool to provide real time classification and pattern matching of the process's binary images. The image path is extracted from the `ThreadInfo` instance after `CreateProcess` kernel event has been captured. :param dict outputs: declared output adapters :param logbook.Logger logger: reference to the logger implementation :param dict config: configuration for this binding """ BaseBinding.__init__(self, outputs, logger) self._path = config.pop('path', None) self._rules = None if not yara: raise BindingError('yara-python package is not installed') if not os.path.exists(self._path) or not os.path.isdir(self._path): raise BindingError('%s rules path does not exist' % self._path) try: for file in glob.glob(os.path.join(self._path, '*.yar')): self._rules = yara.compile(os.path.join(self._path, file)) except yara.SyntaxError as e: raise BindingError("rule compilation error %s" % e)
def test_private_rule(self): """test private rule behaviour""" source = """ private rule PrivateTestRule { strings: $a = "private" condition: $a } rule TestRule { condition: PrivateTestRule } """ rules = yara.compile(source=source) res = rules.match_data("private rule ftw") self.assertTrue("main" in res) self.assertEqual(len(res["main"]), 1) self.assertTrue(res["main"][0]["rule"], "TestRule") res = rules.match_data("aaa") self.assertTrue("main" not in res)
def yara_scan(self): ''' { 'tags': ['foo', 'bar'], 'matches': True, 'namespace': 'default', 'rule': 'my_rule', 'meta': {}, 'strings': [(81L, '$a', 'abc'), (141L, '$b', 'def')] } ''' try: self.yara_scan_result = [] yara_uncompiled_rules = static_conf["yara_uncompiled_rules"] yara_compiled_rules = static_conf["yara_compiled_rules"] yara_rules_list = [] # load rules if yara_uncompiled_rules: yara_rules_list.append(yara.compile(filepaths = yara_uncompiled_rules)) if yara_compiled_rules: yara_rules_list.extend([yara.load(os.path.join(yara_compiled_rules,item)) for item in os.listdir(yara_compiled_rules)]) # match yara rules for rules in yara_rules_list: matches = rules.match(self.filepath) self.yara_scan_result.extend([{"namespace":match.namespace,"rule":match.rule,"meta":match.meta} for match in matches]) except Exception as e: self.logger.exception('%s: %s' % (Exception, e))
def test_meta_is_exported(self): """test meta data export""" source = """ rule TestMeta { meta: signature = "this is my sig" excitement = 10 want = true strings: $test_string = " bird" condition: $test_string } """ rules = yara.compile(source=source) res = rules.match_data("mocking bird") self.assertTrue("main" in res) self.assertEqual(len(res["main"]), 1) meta = res["main"][0]["meta"] self.assertEqual(meta["excitement"], 10) self.assertEqual(meta["signature"], "this is my sig") self.assertEqual(meta["want"], True) res = rules.match_data("no mocking this time") self.assertTrue("main" not in res)
def test_compile_sources(self): """compile sources""" filepath = os.path.join(RULES_ROOT, "meta.yar") with open(filepath, "rb") as f: source = f.read() rule = yara.compile(sources=dict(test_ns=source)) self.assert_scan(rule)
def __init__(self, *args, **kwargs): """Scan using yara signatures.""" super(YaraScanMixin, self).__init__(*args, **kwargs) # Compile the yara rules in advance. if self.plugin_args.yara_expression: self.rules_source = self.plugin_args.yara_expression self.rules = yara.compile(source=self.rules_source) elif self.plugin_args.binary_string: self.compile_rule( 'rule r1 {strings: $a = {%s} condition: $a}' % self.plugin_args.binary_string) elif self.plugin_args.string: self.compile_rule( 'rule r1 {strings: $a = "%s" condition: $a}' % self.plugin_args.string) elif self.plugin_args.yara_file: self.compile_rule(open(self.plugin_args.yara_file).read()) elif not self.ignore_required: raise plugin.PluginError("You must specify a yara rule file or " "string to match.")
def yaradisk(path,server,rule,silent): """ Yara file/directory object scan module """ rule_text=fetchyararule(server,rule) if not silent: print '\nPulling ' +rule+ ' from ' +server+ '\n' print rule_text + '\n' print '\nScanning ' +path+ '\n' rule_bin=yara.compile(sources={'namespace':rule_text}) for root, dirs, filenames in os.walk(path): for name in filenames: try: file_path=os.path.join(root,name) matches=rule_bin.match(filepath=file_path) if matches: payload={"rulename":matches[0], "filename":file_path, "module":'yaradisk', "hostname":os.uname()[1]} if not silent: print payload p=post('http://'+server+':'+str(server_port)+'/putfile',data=payload) except: continue
def scan_file(self): """docstring for scan_file""" rules = yara.compile(filepaths=self.rules_files) matches = rules.match(data=self.file_content) for m in matches: self.results.append(m.rule)
def yara_malware(filename: str) -> list: rules = yara.compile("rules/malware_index.yar") return rules.match(filename)
def yara_antitricks(filename: str) -> list: rules = yara.compile("rules/Antidebug_AntiVM_index.yar") return rules.match(filename)
print "[*] Converting Markdown README to reStructuredText" rst = pypandoc.convert_file('README.md', 'rst') with open('README.rst', 'w+', encoding='utf-8') as f: f.write(rst) print "[*] Finished converting to README.rst ({} bytes)".format(len(rst)) yara_files = {} for root, dirnames, filenames in os.walk(rules_dir): for filename in fnmatch.filter(filenames, '*.yara'): path = os.path.join(root, filename) yara_files[path] = path # print yara_files print "[*] Compiling {} Yara rule files".format(len(yara_files)) rules = yara.compile(filepaths=yara_files) rules.save(compiled_rules_path) count = 0 for _ in rules: count += 1 print "[*] Saved {} rules to {}".format(count, compiled_rules_path) # print "[*] Registering ..." # os.system("python setup.py register") # print "[*] Cleaning up ..." # os.remove('README.rst') print "[*] Done."
def __init__(self, pattern=None, **_): try: yara.compile(source=pattern) except (yara.SyntaxError, yara.Error) as e: raise ValidationError('{0:s} is not a valid Yara rule: {1!s}'.format(pattern, e))
#!/usr/bin/python import sys, yara, commands rules = yara.compile(sys.argv[1]) data = open(sys.argv[2], 'rb').read() matches = rules.match(data=data) isupx = [m for m in matches if m.rule.startswith("UPX")] if isupx: outp = commands.getoutput("upx -d %s" % sys.argv[2]) print outp
def yara_scan(fi): """ Scan selected region (the whole file if not selected) with YARA """ if yara_python_not_installed: print "yara-python is not installed." print "Please install it with 'python -m pip install yara-python' and restart FileInsight." return cp = ctypes.windll.kernel32.GetACP() cp = "cp%d" % cp num_file = fi.getDocumentCount() if num_file < 2: if num_file == 1: print "Please open a file to be scanned and a YARA rule file before using YARA scan plugin." return file_list = "" for i in range(num_file): fi.activateDocumentAt(i) file_list += "%s\r\n" % fi.getDocumentName().decode(cp).encode("utf-8") # Do not show command prompt window startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW # Execute file_comparison_dialog.py to show GUI # GUI portion is moved to send_to.py to avoid hangup of FileInsight p = subprocess.Popen(["python", "yara_scan_dialog.py"], startupinfo=startupinfo, stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdout_data, stderr_data = p.communicate(input=file_list) if stdout_data == "": return (scanned_file_index, rule_file_index) = stdout_data.split() time_start = time.time() fi.activateDocumentAt(int(scanned_file_index)) length_sel = fi.getSelectionLength() offset = fi.getSelectionOffset() if length_sel > 0: buf = fi.getSelection() buf_len = length_sel else: buf = fi.getDocument() buf_len = fi.getLength() offset = 0 if buf == "": return fi.activateDocumentAt(int(rule_file_index)) rule = fi.getDocument() try: y = yara.compile(source=rule) except Exception as e: print "Error: invalid YARA rule" print e return if length_sel > 0: print "Scan from offset %s to %s.\r\n" % (hex(offset), hex(offset + buf_len - 1)) else: print "Scan the whole file.\r\n" match = y.match(data=buf) if len(match) == 0: print "No YARA rule matched." print "Elapsed time (scan): %f (sec)" % (time.time() - time_start) return num_hits = 0 bookmark_start = [] bookmark_end = [] for m in match: prev_string = "" for i in range(0, len(m.strings)): if is_printable(m.strings[i][2]): print "Offset: 0x%x rule: %s tag: %s identifier: %s matched: %s" % ( offset + m.strings[i][0], m.rule, " ".join(m.tags), m.strings[i][1], re.sub("[\r\n\v\f]", "", m.strings[i][2])) else: print "Offset: 0x%x rule: %s tag: %s identifier: %s matched: %s (hex)" % ( offset + m.strings[i][0], m.rule, " ".join(m.tags), m.strings[i][1], binascii.hexlify(m.strings[i][2])) if num_hits > 0 and m.strings[i][ 1] == prev_string and offset + m.strings[i][ 0] <= bookmark_end[-1]: bookmark_end[-1] = offset + m.strings[i][0] + len( m.strings[i][2]) else: bookmark_start.append(offset + m.strings[i][0]) bookmark_end.append(offset + m.strings[i][0] + len(m.strings[i][2])) prev_string = m.strings[i][1] num_hits += 1 print "\r\nElapsed time (scan): %f (sec)" % (time.time() - time_start) time_start = time.time() fi.activateDocumentAt(int(scanned_file_index)) for i in range(0, len(bookmark_start)): fi.setBookmark(bookmark_start[i], bookmark_end[i] - bookmark_start[i], hex(bookmark_start[i]), "#aaffaa") if num_hits == 1: print "Added a bookmark to the search hit." elif num_hits > 1: print "Added bookmarks to the search hits." if num_hits > 0: print "Elapsed time (bookmark): %f (sec)" % (time.time() - time_start)
def yara_crypto(filename: str) -> list: rules = yara.compile("rules/Crypto_index.yar") return rules.match(filename)
def init_yara(): """Generates index for yara signatures.""" categories = ("binaries", "urls", "memory", "CAPE", "macro", "monitor") log.debug("Initializing Yara...") # Generate root directory for yara rules. yara_root = os.path.join(CUCKOO_ROOT, "data", "yara") # Loop through all categories. for category in categories: # Check if there is a directory for the given category. category_root = os.path.join(yara_root, category) if not os.path.exists(category_root): log.warning("Missing Yara directory: %s?", category_root) continue rules, indexed = {}, [] for category_root, _, filenames in os.walk(category_root, followlinks=True): for filename in filenames: if not filename.endswith((".yar", ".yara")): continue filepath = os.path.join(category_root, filename) rules["rule_%s_%d" % (category, len(rules))] = filepath indexed.append(filename) # Need to define each external variable that will be used in the # future. Otherwise Yara will complain. externals = {"filename": ""} while True: try: File.yara_rules[category] = yara.compile(filepaths=rules, externals=externals) break except yara.SyntaxError as e: bad_rule = str(e).split(".yar")[0] + ".yar" log.debug(f"Trying to delete bad rule: {bad_rule}") if os.path.basename(bad_rule) in indexed: for k, v in rules.items(): if v == bad_rule: del rules[k] indexed.remove(os.path.basename(bad_rule)) print("Deleted broken yara rule: {}".format( bad_rule)) break else: break except yara.Error as e: print( "There was a syntax error in one or more Yara rules: %s" % e) log.error( "There was a syntax error in one or more Yara rules: %s" % e) break if category == "memory": mem_rules = yara.compile(filepaths=rules, externals=externals) mem_rules.save(os.path.join(yara_root, "index_memory.yarc")) """ with open(os.path.join(yara_root, "index_memory.yarc"), "w") as f: for filename in sorted(indexed): f.write('include "%s"\n' % os.path.join(category_root, filename)) """ indexed = sorted(indexed) for entry in indexed: if (category, entry) == indexed[-1]: log.debug("\t `-- %s %s", category, entry) else: log.debug("\t |-- %s %s", category, entry)
def calculate(self): if not has_yara: debug.error("You must install yara") addr_space = utils.load_as(self._config) if not self.is_valid_profile(addr_space.profile): debug.error("This command does not support the selected profile.") rules = yara.compile(sources = self.signatures) for task in self.filter_tasks(tasks.pslist(addr_space)): task_space = task.get_process_address_space() # We must have a process AS if not task_space: continue for vad, process_space in task.get_vads(): if obj.Object("_IMAGE_DOS_HEADER", offset = vad.Start, vm = process_space).e_magic != 0x5A4D: continue data = process_space.zread(vad.Start, vad.Length) # check for the signature with YARA, both hits must be present matches = rules.match(data = data) if len(matches) < 2: continue try: dos_header = obj.Object("_IMAGE_DOS_HEADER", offset = vad.Start, vm = task_space) nt_header = dos_header.get_nt_header() except (ValueError, exceptions.SanityCheckException): continue # There must be more than 2 sections if nt_header.FileHeader.NumberOfSections < 2: continue # Get the last PE section's data sections = list(nt_header.get_sections(False)) last_sec = sections[-1] last_sec_data = task_space.zread( (last_sec.VirtualAddress + vad.Start), last_sec.Misc.VirtualSize ) success = self.check_matches(task_space, vad, matches, last_sec_data) if not success: continue success = self.scan_key(task_space) if not success: continue yield task, vad, self.params
def compile(self) -> yara.Rules: yara_files = self._collect_yara_files() self.rules = yara.compile(filepaths=yara_files) return self.rules
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.compiled_pattern = yara.compile(source=self.pattern)
def yara_packer(filename: str) -> list: rules = yara.compile("rules/Packers_index.yar") return rules.match(filename)
ssl_chain=False) else: print("Server not found, quitting...") sys.exit(1) else: if 'default' not in config.keys(): print("No default severs in MISP conf, quitting...") sys.exit(1) else: server = MispServer(url=config['default']['url'], apikey=config['default']['key'], ssl_chain=False) event = server.events.get(args.event) rules = yara.compile(filepath=args.rules) for attr in event.attributes: if attr.type == 'malware-sample': # Ignore zip files value = str(attr.value) if "|" in value: fname = value.split("|")[0] if not fname.endswith(".zip"): data = server.download(attr) check_yara(rules, data, attr.value, args.verbose) else: if args.verbose > 1: print("%s not considered (.zip file)" % (attr.value)) else: data = server.download(attr)
def search(self): memory, offsets = self._get_memory() rules = yara.compile(filepaths=self.get_rules_files()) values = self.yarasearch(memory, offsets, rules) c = YaraSearchResultChooser("Findcrypt results", values) r = c.show()
def scan(filelist, conf=DEFAULTCONF): ruleDir = conf["ruledir"] extlist = conf["fileextensions"] includes = 'includes' in conf and conf['includes'] ruleset = {} rules = parseDir(ruleDir, recursive=True) for r in rules: for ext in extlist: if r.endswith(ext): full_path = os.path.abspath(os.path.join(ruleDir, r)) ruleset[full_path] = full_path break #Ran into a weird issue with file locking, this fixes it goodtogo = False i = 0 yararules = None while not goodtogo: try: yararules = yara.compile(filepaths=ruleset, includes=includes) goodtogo = True except yara.SyntaxError as e: bad_file = os.path.abspath(str(e).split('(')[0]) if bad_file in ruleset: del ruleset[bad_file] print('WARNING: Yara', e) else: print( 'ERROR Yara: Invalid rule in', bad_file, 'but we are unable to remove it from our list. Aborting') print(e) return None matches = [] for m in filelist: #Ran into a weird issue with file locking, this fixes it goodtogo = False i = 0 while not goodtogo and i < 5: try: f = open(m, 'rb') goodtogo = True except Exception as e: print('yara:', e) time.sleep(3) i += 1 try: hit = yararules.match(data=f.read()) except: continue finally: f.close() if hit: hdict = {} for h in hit: if not set(h.tags).intersection(set(conf["ignore-tags"])): hit_dict = { 'meta': h.meta, 'namespace': h.namespace, 'rule': h.rule, 'tags': h.tags, } try: h_key = '{}:{}'.format( hit_dict['namespace'].split('/')[-1], hit_dict['rule']) except IndexError: h_key = '{}'.format(hit_dict['rule']) hdict[h_key] = hit_dict matches.append((m, hdict)) metadata = {} rulelist = list(ruleset) rulelist.sort() metadata["Name"] = NAME metadata["Type"] = TYPE metadata["Rules"] = rulelist return (matches, metadata)
def assertFalseRules(self, rules, data='dummy'): for r in rules: r = yara.compile(source=r) self.assertFalse(r.match(data=data))
def calculate(self): if not has_yara: debug.error("Please install Yara from code.google.com/p/yara-project") addr_space = utils.load_as(self._config) if self._config.YARA_RULES: s = self._config.YARA_RULES # Don't wrap hex or regex rules in quotes if s[0] not in ("{", "/"): s = '"' + s + '"' # Scan for unicode strings if self._config.WIDE: s += "wide" rules = yara.compile(sources = { 'n' : 'rule r1 {strings: $a = ' + s + ' condition: $a}' }) elif self._config.YARA_FILE: rules = yara.compile(self._config.YARA_FILE) else: debug.error("You must specify a string (-Y) or a rules file (-y)") if self._config.KERNEL: # Find KDBG so we know where kernel memory begins. Do not assume # the starting range is 0x80000000 because we may be dealing with # an image with the /3GB boot switch. kdbg = tasks.get_kdbg(addr_space) start = kdbg.MmSystemRangeStart.dereference_as("Pointer") # Modules so we can map addresses to owners mods = dict((mod.DllBase, mod) for mod in modules.lsmod(addr_space)) mod_addrs = sorted(mods.keys()) # There are multiple views (GUI sessions) of kernel memory. # Since we're scanning virtual memory and not physical, # all sessions must be scanned for full coverage. This # really only has a positive effect if the data you're # searching for is in GUI memory. sessions = [] for proc in tasks.pslist(addr_space): sid = proc.SessionId # Skip sessions we've already seen if sid == None or sid in sessions: continue session_space = proc.get_process_address_space() if session_space == None: continue sessions.append(sid) scanner = DiscontigYaraScanner(address_space = session_space, rules = rules) for hit, address in scanner.scan(start_offset = start): module = tasks.find_module(mods, mod_addrs, address) yield (module, address, hit, session_space.zread(address, 1024)) else: for task in self.filter_tasks(tasks.pslist(addr_space)): scanner = VadYaraScanner(task = task, rules = rules) for hit, address in scanner.scan(): yield (task, address, hit, scanner.address_space.zread(address, 1024))
any of ($x*) and #HWORLD > 3 and all of ($r*) } ''' import yara_tools import yara rule = yara_tools.create_rule(name="hello_world") rule.add_meta(key="author", value="@matonis") rule.add_meta(key="purpose", value="Simple 'Hello World'") rule.add_strings(strings="Hello World") rule.add_strings(strings=['Yello, World?', 'Yello, World!'], condition="any of ($IDENTIFIER*)", modifiers=['wide', 'ascii']) rule.add_strings(strings="HelloWorld", identifier="HWORLD", condition="#IDENTIFIER > 3") generated_rule = rule.build_rule() try: compiled_rule = yara.compile(source=generated_rule) print generated_rule print "SUCCESS: IT WORKED!" except Exception as e: print "Failed... oh noes! %s" % e print generated_rule
def testExternals(self): r = yara.compile(source='rule test { condition: ext_int == 15 }', externals={'ext_int': 15}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_int == -15}', externals={'ext_int': -15}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_float == 3.14 }', externals={'ext_float': 3.14}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_float == -0.5 }', externals={'ext_float': -0.5}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_bool }', externals={'ext_bool': True}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str }', externals={'ext_str': ''}) self.assertFalse(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_str }', externals={'ext_str': 'foo'}) self.assertTrue(r.match(data='dummy')) r = yara.compile(source='rule test { condition: ext_bool }', externals={'ext_bool': False}) self.assertFalse(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str contains "ssi" }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /foo/ }', externals={'ext_str': ''}) self.assertFalse(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /foo/ }', externals={'ext_str': 'FOO'}) self.assertFalse(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /foo/i }', externals={'ext_str': 'FOO'}) self.assertTrue(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /ssi(s|p)/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /ppi$/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /ssi$/ }', externals={'ext_str': 'mississippi'}) self.assertFalse(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /^miss/ }', externals={'ext_str': 'mississippi'}) self.assertTrue(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /^iss/ }', externals={'ext_str': 'mississippi'}) self.assertFalse(r.match(data='dummy')) r = yara.compile( source='rule test { condition: ext_str matches /ssi$/ }', externals={'ext_str': 'mississippi'}) self.assertFalse(r.match(data='dummy'))
def match_rule(self, rule, data=None): data = self.file_data if data is None else data myrules = tmp_rule % (rule, self.rules[rule]) yararules = yara.compile(source=myrules) return yararules.match(data=data)
def yaramatch(f): rules = yara.compile("rules/index.yar") matches = rules.match(data=f) return matches
byteArr, fileSize = get_file_bytes_size(filepath) freqList = cal_byteFrequency(byteArr, fileSize) # Shannon entropy ent = 0.0 for freq in freqList: if freq > 0: ent += - freq * math.log(freq, 2) # ent = -ent return [fileSize, ent] ######################################################### yara_path="./peid.yara" rules = yara.compile(filepath=yara_path) ######################################################### def packer_check(queue): while queue.empty() != True: get_tuple= queue.get() sample_path=get_tuple[0] flags = get_tuple[1] if flags==1: print("FSG")
continue if 'private rule' in value: # private rules need some more rewriting priv_rules = re.findall('private rule (\w+)', value, flags=re.MULTILINE) for priv_rule in priv_rules: value = re.sub(priv_rule, 'misp_e{}_{}'.format(event_id, priv_rule), value, flags=re.MULTILINE) # compile the yara rule to confirm it's validity # if valid, ignore duplicate rules try: attr_cnt += 1 yara.compile(source=value) yara_rules.append(value) # print("Rule e{} a{} OK".format(event_id, attribute_id)) except yara.SyntaxError as e: attr_cnt_invalid += 1 # print("Rule e{} a{} NOK - {}".format(event_id, attribute_id, e)) except yara.Error as e: attr_cnt_invalid += 1 print(e) import traceback print(traceback.format_exc()) # remove duplicates - process the full yara rule list and process errors to eliminate duplicate rule names all_yara_rules = '\n'.join(yara_rules) while True: try:
def clean(self): try: yara.compile(source=self.pattern) except (yara.SyntaxError, yara.Error) as e: raise IndicatorValidationError( "Yara compilation error: {}".format(e))
def measure(yara_rule_string, cycles, progress, show_score=True, c_duration=0, rule_name="", alert_diff=0, single_rule=""): """ Measure rule performance :param rule: the YARA rule to test :param cycles: number of iterations over the sample set :param show_score: show the performance score :param c_duration: duration of the calibration run :return duration: duration in seconds :return count: count of samples in the given samples folders """ try: y = yara.compile(source=yara_rule_string, externals={ 'filename': "", 'filepath': "", 'extension': "", 'filetype': "", 'md5': "", }) except Exception as e: Log.error("Error compiling YARA rule '%s' : %s" % ( rule_name, e )) return 0,0,0 #Log.info("Scanning sample set %d times with rule: %s" % (cycles, rule_name )) min_duration=9999999999999999 max_duration=0 diff_perc = 0 count_mega_slow = 0 for _ in range(cycles): # do garbage collection to avoid that it happens during benchmarking gc.collect() start = time.time() for sample in samples: try: matches = y.match(data=sample, externals={ 'filename': "", 'filepath': "", 'extension': "", 'filetype': "", 'md5': "", }) except Exception as e: Log.error("Error matching YARA rule '%s' : %s" % ( rule_name, e)) traceback.print_exc() # TODO: sys.exit or not??? #sys.exit(1) end = time.time() duration = end - start if duration < min_duration: min_duration = duration #print("New min: ", duration) if duration > max_duration: max_duration = duration #print("New max: ", duration) # If a calibration duration has been evaluated if c_duration > 0: diff_perc = ( (min_duration / c_duration -1)*100 ) # skip test if this scan was too fast if not slow_mode and diff_perc < alert_diff: progress.console.print("[INFO ] Rule \"%s\" is fast enough, not measuring any further due to fast mode, diff %0.4f %% below alerting level: %0.4f %%" % (rule_name, diff_perc, alert_diff )) return 0,0,0 # stop test if this scan was mega slow for 10th time => warning because alert_diff is high enough if not slow_mode and diff_perc > 2 * alert_diff: count_mega_slow += 1 if count_mega_slow > 10: break if c_duration and not rule_name == "Baseline": if diff_perc > alert_diff: log_warning_rich("Rule \"%s\" slows down a search with %d rules by %0.4f %% (Measured by best of %d runs)" % (rule_name, num_calib_rules, diff_perc , cycles ), single_rule) else: if show_score: progress.console.print("[INFO ] Rule: \"%s\" - Best of %d - duration: %.4f s (%0.4f s, %0.4f %%)" % (rule_name, cycles, min_duration, (min_duration-c_duration), diff_perc )) else: progress.console.print("[INFO ] Rule: \"%s\" - best of %d - duration: %.4f s" % (rule_name, cycles, min_duration)) return min_duration, count, diff_perc
def search(self): memory, offsets = self._get_memory() rules = yara.compile(YARARULES_CFGFILE) values = self.yarasearch(memory, offsets, rules) c = YaraSearchResultChooser("Findcrypt results", values) r = c.show()
def yara_cve(filename: str) -> list: rules = yara.compile("rules/CVE_Rules_index.yar") return rules.match(filename)
def scan(self): def string_printable(line): line = str(line) new_line = '' for c in line: if c in printstring.printable: new_line += c else: new_line += '\\x' + c.encode('hex') return new_line # This means users can just drop or remove rule files without # having to worry about maintaining the index. # TODO: make paths absolute. # TODO: this regenerates the file at every run, perhaps we # could find a way to optimize this. def rule_index(): with open('data/yara/index.yara', 'w') as rules_index: for rule_file in os.listdir('data/yara'): # Skip if the extension is not right, could cause problems. if not rule_file.endswith( '.yar') and not rule_file.endswith('.yara'): continue # Skip if it's the index itself. if rule_file == 'index.yara': continue # Add the rule to the index. line = 'include "{0}"\n'.format(rule_file) rules_index.write(line) return 'data/yara/index.yara' arg_rule = self.args.rule arg_scan_all = self.args.all arg_tag = self.args.tag # If no custom ruleset is specified, we use the default one. if not arg_rule: arg_rule = rule_index() # Check if the selected ruleset actually exists. if not os.path.exists(arg_rule): self.log('error', "No valid Yara ruleset at {0}".format(arg_rule)) return # Compile all rules from given ruleset. rules = yara.compile(arg_rule) files = [] # If there is a session open and the user didn't specifically # request to scan the full repository, we just add the currently # opened file's path. if __sessions__.is_set() and not arg_scan_all: files.append(__sessions__.current.file) # Otherwise we loop through all files in the repository and queue # them up for scan. else: self.log('info', "Scanning all stored files...") db = Database() samples = db.find(key='all') for sample in samples: files.append(sample) for entry in files: if entry.size == 0: continue self.log('info', "Scanning {0} ({1})".format(entry.name, entry.sha256)) # Check if the entry has a path attribute. This happens when # there is a session open. We need to distinguish this just for # the cases where we're scanning an opened file which has not been # stored yet. if hasattr(entry, 'path'): entry_path = entry.path # This should be triggered only when scanning the full repository. else: entry_path = get_sample_path(entry.sha256) # Check if the file exists before running the yara scan. if not os.path.exists(entry_path): self.log( 'error', "The file does not exist at path {0}".format(entry_path)) return rows = [] tag_list = [] for match in rules.match(entry_path): # Add a row for each string matched by the rule. for string in match.strings: rows.append([ match.rule, string_printable(string[1]), string_printable(string[0]), string_printable(string[2]) ]) # Add matching rules to our list of tags. # First it checks if there are tags specified in the metadata # of the Yara rule. match_tags = match.meta.get('tags') # If not, use the rule name. # TODO: as we add more and more yara rules, we might remove # this option and only tag the file with rules that had # tags specified in them. if not match_tags: match_tags = match.rule # Add the tags to the list. tag_list.append([entry.sha256, match_tags]) if rows: header = ['Rule', 'String', 'Offset', 'Content'] self.log('table', dict(header=header, rows=rows)) # If we selected to add tags do that now. if rows and arg_tag: db = Database() for tag in tag_list: db.add_tags(tag[0], tag[1]) # If in a session reset the session to see tags. if __sessions__.is_set() and not arg_scan_all: self.log('info', "Refreshing session to update attributes...") __sessions__.new(__sessions__.current.file.path)
(end_time - start_time) )) if __name__ == "__main__": logger.info("Compile Yara Rules") try: # Update the yara rules index yara_index(conf['yara']['rule_path'], conf['yara']['blacklist'], conf['yara']['test_rules']) # Compile the yara rules we will use to match pastes index_file = os.path.join(conf['yara']['rule_path'], 'index.yar') rules = yara.compile(index_file) except Exception as e: print("Unable to Create Yara index: ", e) sys.exit() # Create Queue to hold paste URI's q = Queue() processes = [] # Threads for i in range(5): m = multiprocessing.Process(target=paste_scanner) # Add new process to list so we can run join on them later. processes.append(m) m.start()