def on_call(self, call, process): if call["api"] == "CreateServiceA" or call["api"] == "CreateServiceW": starttype = call["arguments"]["start_type"] servicename = call["arguments"]["service_name"] servicepath = call["arguments"]["filepath"] if starttype < 3: self.mark( service_name=servicename, service_path=servicepath, ) elif call["status"]: regkey = call["arguments"]["regkey"] regvalue = call["arguments"]["value"] in_whitelist = False for whitelist in self.whitelists: if re.match(whitelist, regkey, re.IGNORECASE): in_whitelist = True break if not in_whitelist: for indicator in self.regkeys_re: if re.match( indicator, regkey, re.IGNORECASE ) and regvalue != "c:\\program files\\java\\jre7\\bin\jp2iexp.dll": self.mark( reg_key=regkey, reg_value=regvalue, )
def on_call(self, call, process): processname = process["process_name"].lower() if processname == "mshta.exe" or processname == "rundll32.exe": if call["api"] == "NtCreateUserProcess" or call[ "api"] == "CreateProcessInternalW": cmdline = self.get_argument(call, "CommandLine") if cmdline: for pat in self.patterns: if re.match(pat, cmdline): self.score += 1 self.data.append({"cmdline": cmdline}) if call["api"] == "ShellExecuteExW": show = self.get_argument(call, "Show") param = self.get_argument(call, "Parameters") if show and param: if int(show, 10) == 0: for pat in self.patterns: if re.match(pat, param): self.score += 1 self.data.append({"parameter": param}) if call["api"] == "NtWriteFile" and processname == "rundll32.exe" and self.score > 3: buff = self.get_argument(call, "CommandLine") handle = self.get_argument(call, "HandleName") if buff: if buff.startswith("MZ"): self.score += 2 self.data.append({"payload": handle})
def on_call(self, call, process): if call["api"] == "RegSetValueExA": # Autorun registry / filesystem behavior key = self.get_argument(call, "FullName").lower() if "\\software\\microsoft\\windows\\currentversion\\run\\" in key: buf = self.get_argument(call, "Buffer").lower() if re.match(r"^[A-Z]:\\ProgramData\\\w+\\\w+\.exe$", buf): self.vawtrakauto = True elif call["api"] == "NtCreateEvent" or call["api"] == "NtOpenEvent": buf = self.get_argument(call, "EventName") if re.match(r"^\{[0-9A-F]{8}(-[0-9A-F]{4}){3}-[0-9A-F]{12}\}$", buf): self.eventtrigger = True else: self.eventtrigger = False elif call["api"] == "CreateThread" or call["api"] == "NtOpenProcess": if self.eventtrigger: self.malscore += 2 self.lastcall = call["api"] # Reset event trigger if the current API isn't Nt[Create|Open]Event if call["api"] not in ["NtCreateEvent", "NtOpenEvent"]: self.eventtrigger = False
def on_call(self, call, process): if call["api"] == "CreateServiceA" or call["api"] == "CreateServiceW": starttype = call["arguments"]["start_type"] servicename = call["arguments"]["service_name"] servicepath = call["arguments"]["filepath"] if starttype < 3: self.mark( service_name=servicename, service_path=servicepath, ) elif call["status"]: regkey = call["arguments"]["regkey"] regvalue = call["arguments"]["value"] in_whitelist = False for whitelist in self.whitelists: if re.match(whitelist, regkey, re.IGNORECASE): in_whitelist = True break if not in_whitelist: for indicator in self.regkeys_re: if re.match(indicator, regkey, re.IGNORECASE) and regvalue != "c:\\program files\\java\\jre7\\bin\jp2iexp.dll": self.mark( reg_key=regkey, reg_value=regvalue, )
def on_call(self, call, process): if call["api"] == "RegSetValueExA": # Autorun registry / filesystem behavior key = call["arguments"]["regkey"].lower() if "\\software\\microsoft\\windows\\currentversion\\run\\" in key: buf = call["arguments"]["value"] try: #If integer, lets just continue intval = int(buff) return except: pass #Do nothing# buf = (buf).encode('ascii', 'ignore').decode('ascii') buf = buf.lower() if re.match(r"^[A-Z]:\\ProgramData\\\w+\\\w+\.exe$", buf): self.vawtrakauto = True elif call["api"] == "NtCreateEvent" or call["api"] == "NtOpenEvent": buf = self.get_argument(call, "EventName") if re.match(r"^\{[0-9A-F]{8}(-[0-9A-F]{4}){3}-[0-9A-F]{12}\}$", buf): self.eventtrigger = True else: self.eventtrigger = False elif call["api"] == "CreateThread" or call["api"] == "NtOpenProcess": if self.eventtrigger: self.malscore += 2 self.lastcall = call["api"] # Reset event trigger if the current API isn't Nt[Create|Open]Event if call["api"] not in ["NtCreateEvent", "NtOpenEvent"]: self.eventtrigger = False
def on_call(self, call, process): continueChain = False if call["status"]: if call["api"] == "LdrGetProcedureAddress": resolved = self.get_argument(call, "FunctionName") if resolved and resolved == "IsWow64Process": continueChain = True elif call["api"] == "NtCreateEvent": if self.lastapi == "LdrGetProcedureAddress" and self.chain: event = self.get_argument(call, "EventName") if event and re.match("^[0-9A-F]{32}$", event): continueChain = True elif call["api"] == "NtCreateSection": if self.lastapi == "NtCreateEvent" and self.chain: attribs = self.get_argument(call, "ObjectAttributes") if attribs and re.match("^[0-9A-F]{32}$", attribs): continueChain = True elif call["api"] == "CreateThread": if self.lastapi == "NtCreateSection" and self.chain: return True self.chain = continueChain self.lastapi = call["api"]
def on_call(self, call, process): if call["api"] == "NtCreateFile": desiredaccess = int(self.get_argument(call, "DesiredAccess"), 16) if desiredaccess and (desiredaccess & 0x80100080 or desiredaccess & 0x00110081): filename = self.get_argument(call, "FileName") if filename and re.match(self.filepattern, filename, re.IGNORECASE): self.filematch = True self.filenames.append(filename) if call["api"] == "NtOpenFile": desiredaccess = int(self.get_argument(call, "DesiredAccess"), 16) if desiredaccess and desiredaccess & 0x00020080: filename = self.get_argument(call, "FileName") if filename and re.match(self.filepattern, filename, re.IGNORECASE): self.filematch = True self.filenames.append(filename) if call["api"] == "NtReadFile": filename = self.get_argument(call, "FileName") if filename and re.match(self.filepattern, filename, re.IGNORECASE): self.filematch = True self.filenames.append(filename)
def __match_to_known_token(word): # URL if re.match(r"^(https?:\/\/(?:www\.|(?!www))[^\s\.]+\.[^\s]{2,}|www\.[^\s]+\.[^\s]{2,})$", word): return TOKENS["URL_TOKEN"] # FLOAT if re.match(r'^([0-9]+\.)[0-9]+$', word): return TOKENS["FLOAT_TOKEN"]
def filter_allowed_disallowed(subjects, allowed_patterns, disallowed_patterns): """Filter a list using allowed and disallowed patterns. :param list subjects: A list of strings to filter. :param allowed_patterns: A list of re2-compatible patterns to allow. If empty, all subjects are allowed (see next). :param disallowed_patterns: A list of re2-compatible patterns to reject. A more-specific pattern here may override a less-specific allowed pattern. If empty, all allowed subjects will pass. """ ret = [] for subject in subjects: allowed = True if allowed_patterns: allowed = False for pattern in allowed_patterns: if re2.match(pattern, subject): allowed = True break if allowed and disallowed_patterns: for pattern in disallowed_patterns: if re2.match(pattern, subject): allowed = False break if allowed: ret.append(subject) return ret
def test_bug_527371(self): # bug described in patches 527371/672491 self.assertEqual(re.match(r'(a)?a', 'a').lastindex, None) self.assertEqual(re.match(r'(a)(b)?b', 'ab').lastindex, 1) self.assertEqual(re.match(r'(?P<a>a)(?P<b>b)?b', 'ab').lastgroup, 'a') self.assertEqual(re.match("(?P<a>a(b))", "ab").lastgroup, 'a') self.assertEqual(re.match("((a))", "a").lastindex, 1)
def on_call(self, call, process): if call["api"] == "RtlDecompressBuffer": ubuff = self.get_argument(call, "UncompressedBuffer") if ubuff and ubuff.startswith("MZ"): self.rtldecmz = True self.score += 1 if call["api"] == "CreateProcessInternalW": appname = self.get_argument(call, "ApplicationName") if appname: if re.match(self.msbuild, appname) or re.match( self.regasm, appname): flags = int(self.get_argument(call, "CreationFlags"), 16) # CREATE_SUSPENDED|CREATE_NO_WINDOW if flags & 0x4 or flags & 0x08000004: self.score += 2 if call["api"] == "CryptHashData": buff = self.get_argument(call, "Buffer") if buff: if buff.startswith("MZ"): self.cryptmz = True self.score += 1 if buff == "Nativ3M3thodsKey": self.score += 1
def test_bug_527371(self): # bug described in patches 527371/672491 self.assertEqual(re.match(r'(a)?a','a').lastindex, None) self.assertEqual(re.match(r'(a)(b)?b','ab').lastindex, 1) self.assertEqual(re.match(r'(?P<a>a)(?P<b>b)?b','ab').lastgroup, 'a') self.assertEqual(re.match("(?P<a>a(b))", "ab").lastgroup, 'a') self.assertEqual(re.match("((a))", "a").lastindex, 1)
def test_bug_448951(self): # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) for op in '','?','*': self.assertEqual(re.match(r'((.%s):)?z'%op, 'z').groups(), (None, None)) self.assertEqual(re.match(r'((.%s):)?z'%op, 'a:z').groups(), ('a:', 'a'))
def test_getlower(self): import _sre self.assertEqual(_sre.getlower(ord('A'), 0), ord('a')) self.assertEqual(_sre.getlower(ord('A'), re.LOCALE), ord('a')) self.assertEqual(_sre.getlower(ord('A'), re.UNICODE), ord('a')) self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC")
def test_bug_448951(self): # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) for op in '', '?', '*': self.assertEqual( re.match(r'((.%s):)?z' % op, 'z').groups(), (None, None)) self.assertEqual( re.match(r'((.%s):)?z' % op, 'a:z').groups(), ('a:', 'a'))
def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC") self.assertEqual(re.match(r"(a\s[^a])", "a b", re.I).group(1), "a b") self.assertEqual(re.match(r"(a\s[^a]*)", "a bb", re.I).group(1), "a bb") self.assertEqual(re.match(r"(a\s[abc])", "a b", re.I).group(1), "a b") self.assertEqual(re.match(r"(a\s[abc]*)", "a bb", re.I).group(1), "a bb") self.assertEqual(re.match(r"((a)\s\2)", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s\2*)", "a aa", re.I).group(1), "a aa") self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
def test_search_star_plus(self): self.assertEqual(re.search('x*', 'axx').span(0), (0, 0)) self.assertEqual(re.search('x*', 'axx').span(), (0, 0)) self.assertEqual(re.search('x+', 'axx').span(0), (1, 3)) self.assertEqual(re.search('x+', 'axx').span(), (1, 3)) self.assertEqual(re.search('x', 'aaa'), None) self.assertEqual(re.match('a*', 'xxx').span(0), (0, 0)) self.assertEqual(re.match('a*', 'xxx').span(), (0, 0)) self.assertEqual(re.match('x*', 'xxxa').span(0), (0, 3)) self.assertEqual(re.match('x*', 'xxxa').span(), (0, 3)) self.assertEqual(re.match('a+', 'xxx'), None)
def test_bug_418626(self): # bugs 418626 at al. -- Testing Greg Chapman's addition of op code # SRE_OP_MIN_REPEAT_ONE for eliminating recursion on simple uses of # pattern '*?' on a long string. self.assertEqual(re.match('.*?c', 10000*'ab'+'cd').end(0), 20001) self.assertEqual(re.match('.*?cd', 5000*'ab'+'c'+5000*'ab'+'cde').end(0), 20003) self.assertEqual(re.match('.*?cd', 20000*'abc'+'de').end(0), 60001) # non-simple '*?' still used to hit the recursion limit, before the # non-recursive scheme was implemented. self.assertEqual(re.search('(a|b)*?c', 10000*'ab'+'cd').end(0), 20001)
def on_complete(self): indicators = [ ".*\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\SharedTaskScheduler\\\\.*", ] whitelists = [ ".*\\\\Software\\\\(Wow6432Node\\\\)?Classes\\\\clsid\\\\{CAFEEFAC-0017-0000-FFFF-ABCDEFFEDCBA}\\\\InprocServer32\\\\.*", #".*\\\\Software\\\\(Wow6432Node\\\\)?Classes\\\\clsid\\\\[^\\\\]*\\\\InprocServer32\\\\ThreadingModel$", ] for indicator in indicators: match_key = self.check_write_key(pattern=indicator, regex=True, all=True) if match_key: for match in match_key: in_whitelist = False for entry in whitelists: if re.match(entry, match, re.IGNORECASE): in_whitelist = True break if not in_whitelist: data = self.registry_writes.get(match, "unknown") if data.lower( ) != "c:\\program files\\java\\jre7\\bin\jp2iexp.dll": self.data.append({"key": match}) self.data.append({"data": data}) self.found_autorun = True indicators = [".*\\\\WINDOWS\\\\Tasks\\\\.*"] for indicator in indicators: if "dropped" in self.results and len( self.results.get("dropped", [])): for drop in self.results["dropped"]: for path in drop.get("guest_paths", []) or []: if re.match(indicator, path, re.IGNORECASE): self.data.append({"file": path}) self.found_autorun = True match_file = self.check_write_file(pattern=indicator, regex=True, all=True) if match_file: for match in match_file: self.data.append({"file": match}) self.found_autorun = True taskpat = ".*schtasks(\.exe)?.*/CREATE.*/SC\s+.*" tasked = self.check_executed_command(pattern=taskpat, regex=True) if tasked: self.found_autorun = True self.data.append({"task": tasked}) return self.found_autorun
def on_complete(self): for filepath in self.get_files(): if re.match(USERDIR + r'.', filepath, re.IGNORECASE) and not re.match( USERDIR + r'AppData', filepath, re.IGNORECASE): self.mark_ioc("file", filepath) if self.has_marks(): self.description = self.description % len(self.marks) self.severity = 1 + int(round(math.log(len(self.marks)))) return True return False
def validate_result(self): pattern = r"^[a-zA-Z0-9-_'.() ]+$" guid_pattern = r"(\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}" # noqa: E501 result = re.match(pattern, self.text, re.IGNORECASE | re.UNICODE) if result is not None: if len(self.text) > 16: if re.match(guid_pattern, self.text, re.IGNORECASE | re.UNICODE) is not None: return False return True return False
def twitter_sentiment_token_matching(token): """Special token matching function for twitter sentiment data.""" if 'URL_TOKEN' in SPECIAL_TOKENS and re.match(r'https?:\/\/[^\s]+', token): return SPECIAL_TOKENS['URL_TOKEN'] if 'POS_EM_TOKEN' in SPECIAL_TOKENS and re.match(r':-?(\)|D|p)+', token): return SPECIAL_TOKENS['POS_EM_TOKEN'] if 'NEG_EM_TOKEN' in SPECIAL_TOKENS and re.match(r':-?(\(|\\|/)+', token): return SPECIAL_TOKENS['NEG_EM_TOKEN'] if 'USER_TOKEN' in SPECIAL_TOKENS and re.match( r'(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9]+)', token): return SPECIAL_TOKENS['USER_TOKEN'] if 'HEART_TOKEN' in SPECIAL_TOKENS and re.match(r'<3+', token): return SPECIAL_TOKENS['HEART_TOKEN']
def test_re_escape(self): p = "" # This had to change from the original test of range(0,256) # because we can't support non-ascii non-utf8 strings for i in range(0, 128): p = p + chr(i) self.assertEqual(re.match(re.escape(chr(i)), chr(i)) is not None, True) self.assertEqual(re.match(re.escape(chr(i)), chr(i)).span(), (0,1)) pat = re.compile(re.escape(p)) self.assertEqual(pat.match(p) is not None, True) self.assertEqual(pat.match(p).span(), (0,128))
def test_re_escape(self): p="" # This had to change from the original test of range(0,256) # because we can't support non-ascii non-utf8 strings for i in range(0, 128): p = p + chr(i) self.assertEqual(re.match(re.escape(chr(i)), chr(i)) is not None, True) self.assertEqual(re.match(re.escape(chr(i)), chr(i)).span(), (0,1)) pat=re.compile(re.escape(p)) self.assertEqual(pat.match(p) is not None, True) self.assertEqual(pat.match(p).span(), (0,128))
def delete_junk(sentence_list): """ """ sentences = [] num = len(sentence_list) for s in sentence_list: i = 0 while i < num: s = sentence_list[i] # delete any remaining list numbering match = regex_list_start.match(s) if match: s = s[match.end():] # remove any sentences that consist of just '1.', '2.', etc. match = re.match(r'\A\s*\d+(\.|\))\s*\Z', s) if match: i += 1 continue # remove any sentences that consist of '#1', '#2', etc. match = re.match(r'\A\s*#\d+\s*\Z', s) if match: i += 1 continue # remove any sentences consisting entirely of symbols match = re.match(r'\A\s*[^a-zA-Z0-9]+\s*\Z', s) if match: i += 1 continue # merge isolated age + year if i < num-1: if s.isdigit() and sentence_list[i+1].startswith('y'): s = s + ' ' + sentence_list[i+1] i += 1 # if next sentence starts with 'now measures', merge with current if i < num-1: if sentence_list[i+1].startswith('now measures'): s = s + ' ' + sentence_list[i+1] i += 1 sentences.append(s) i += 1 return sentences
def on_call(self, call, process): if call["api"] == "RegSetValueExA": key = self.get_argument(call, "FullName") if key and re.match(self.configkey, key): value = self.get_argument(call, "ValueName").lower() if value == "autoconfigurl": self.keybuf = self.get_argument(call, "Buffer") elif call["api"] == "NtWriteFile": path = self.get_argument(call, "HandleName") if path and re.match(self.configpath, path): buf = self.get_argument(call, "Buffer") if "user_pref" in buf and "network.proxy.autoconfig_url" in buf: tmp = buf.split("(")[1].split(")")[0].split(",")[1] self.pathbuf = tmp.strip().replace("\"","").replace("'","")
def on_call(self, call, process): # Legacy, modern Dyre doesn't have hardcoded hashes in # CryptHashData anymore iocs = [ "J7dnlDvybciDvu8d46D\\x00", "qwererthwebfsdvjaf+\\x00", ] pipe = [ "\\??\\pipe\\3obdw5e5w4", "\\??\\pipe\\g2fabg5713", ] if call["api"] == "CryptHashData": buf = self.get_argument(call, "Buffer") if buf in iocs: self.cryptoapis = True tmp = re.sub(r"\\x[0-9A-Fa-f]{2}", "", buf) if self.compname in tmp: if re.match("^" + self.compname + "[0-9 ]+$", tmp): self.cryptoapis = True elif call["api"] == "HttpOpenRequestA": buf = self.get_argument(call, "Path") if len(buf) > 10: self.networkapis.add(buf) elif call["api"] == "NtCreateNamedPipeFile": buf = self.get_argument(call, "PipeName") for npipe in pipe: if buf == npipe: self.syncapis = True break return None
def run(self): urls = [] whitelist = [ "^http://(crl|ctldl)\.microsoft\.com/.*", ] if "file" in self.results["target"]: if "PDF" in self.results["target"]["file"]["type"] or self.results["info"]["package"] == "pdf": whitelist.append("^http://.*\.adobe\.com/.*") if "network" in self.results and "http" in self.results["network"]: for req in self.results["network"]["http"]: is_whitelisted = False for white in whitelist: if re.match(white, req["uri"], re.IGNORECASE): is_whitelisted = True if not is_whitelisted and req["uri"] not in urls: urls.append(req["uri"]) for url in urls: self.data.append({"url" : url}) if urls: return True return False
def run(self): # will need to turn this into an evented signature later, as IE will read the existing value of some of these entries # and write them back as the same value reg_indicators = [ ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\ProxyEnable$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\ProxyServer$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\ZoneMap\\\\ProxyBypass$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\ProxyOverride$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\.*", ] whitelist = [ ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\WpadLastNetwork$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\[^\\\\]*\\\\WpadDecisionReason$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\[^\\\\]*\\\\WpadDecisionTime$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\[^\\\\]*\\\\WpadDecision$", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Windows\\\\CurrentVersion\\\\Internet\\ Settings\\\\Wpad\\\\[^\\\\]*\\\\WpadNetworkName$", ] # Get rid of a PDF false positive if "file" in self.results["target"]: if "PDF" in self.results["target"]["file"]["type"]: del reg_indicators[0] for indicator in reg_indicators: matches = self.check_write_key(pattern=indicator, regex=True, all=True) if matches: for match in matches: foundwhite = False for white in whitelist: if re.match(white, match, re.IGNORECASE): foundwhite = True if not foundwhite: return True return False
def run(self): # There are more, but these are the only ones I've observed execs = [ "exe", "scr", ] exts = { "doc": "Word 97-2003 document", "docx": "Word 2007+ document", "xls": "Excel 97-2003 spreadsheet", "xlsx": "Excel 2007+ spreadsheet", "ppt": "PowerPoint 97-2003 file", "pptx": "PowerPoint 2007+ file", "jpeg": "JPEG image", "jpg": "JPG image", "png": "PNG image", "gif": "GIF image", "pdf": "PDF document", "xml": "XML document", } pat = ".*[ _\-\.](?P<FakeExtension>{0})\.(?:{1})".format( "|".join(exts.keys()), "|".join(execs)) if self.results["target"]["category"] == "file": check = re.match(pat, self.results["target"]["file"]["name"]) if check: ext = check.group("FakeExtension") self.description = self.description.format(ext, exts[ext.lower()]) return True return False
def validate_regex(check, key, pattern): """Validates that a dict value matches a regex.""" if key not in check: return if not re.match(pattern, check[key]): raise ValueError('Invalid {0}: {1}'.format(key, check[key]))
def run(self): urls = [] whitelist = [ "^http://(crl|ctldl)\.microsoft\.com/.*", "^http://www\.microsoft\.com/.*\.crl$", "^http://ctldl\.windowsupdate\.com/.*", "^http://go\.microsoft\.com/.*", "^http://officecdn\.microsoft\.com/.*", "^http://officecdn\.microsoft\.com\.edgesuite\.net/.*", "^http://apps\.identrust\.com/.*", "^http://*+\.globalsign\.com/.*", "^http://ocsp\.verisign\.com/*." ] if "file" in self.results["target"]: if "PDF" in self.results["target"]["file"]["type"] or self.results["info"]["package"] == "pdf": whitelist.append("^http://.*\.adobe\.com/.*") if "network" in self.results and "http" in self.results["network"]: for req in self.results["network"]["http"]: is_whitelisted = False for white in whitelist: if re.match(white, req["uri"], re.IGNORECASE): is_whitelisted = True if not is_whitelisted and req["uri"] not in urls: urls.append(req["uri"]) for url in urls: self.data.append({"url" : url}) if urls: return True return False
def doPrivmsg(self, irc, msg): """ Handle everything. The name is misleading. This hook actually gets called for all IRC activity in every channel. """ # We don't handle this if we've been addressed because invalidCommand # will handle it for us. This prevents us from accessing the db twice # and therefore crashing. if (msg.addressed or msg.repliedTo): return channel = msg.args[0] if irc.isChannel(channel) and self.allow_unaddressed_karma: irc = callbacks.SimpleProxy(irc, msg) agent = msg.nick line = msg.args[1].strip() # First try to handle karma commands words = line.split() for word in words: if word[-2:] in self.karma_tokens: self._do_karma( irc, channel, agent, word, line, explicit=False) blacklist = self.registryValue('naked_ping_channel_blacklist') if irc.isChannel(channel) and not channel in blacklist: # Also, handle naked pings for # https://github.com/fedora-infra/supybot-fedora/issues/26 pattern = '\w* ?[:,] ?ping\W*$' if re.match(pattern, line): admonition = self.registryValue('naked_ping_admonition') irc.reply(admonition)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.sf.debug("Received event, " + eventName + ", from " + srcModuleName) if eventData not in self.results.keys(): self.results[eventData] = True else: return None for regexpGrp in regexps.keys(): for regex in regexps[regexpGrp]: bits = re.match(regex, eventData, re.IGNORECASE) if bits is not None: self.sf.info("Matched " + regexpGrp + " in " + eventData) evt = SpiderFootEvent("SOCIAL_MEDIA", regexpGrp + ": " + eventData, self.__name__, event) self.notifyListeners(evt) # Except for Google+, the link includes potential usernames if regexpGrp != "Google+": un = bits.group(1) evt = SpiderFootEvent("USERNAME", bits.group(1), self.__name__, event) self.notifyListeners(evt) return None
def run(self): indicators = [ ".*CYBERGATEUPDATE", ".*\(\(SpyNet\)\).*", ".*Spy-Net.*", ".*X_PASSWORDLIST_X.*", ".*X_BLOCKMOUSE_X.*", #".*PERSIST", # Causes false positive detection on XtremeRAT samples. ".*_SAIR", ] for indicator in indicators: if self.check_mutex(pattern=indicator, regex=True): return True keys = [ ".*\\\\SpyNet\\\\.*", ] whitelist = [ ".*\\\\SOFTWARE\\\\Policies\\\\Microsoft\\\\Windows Defender\\\\.*", ] for key in keys: keymatch = self.check_write_key(pattern=key, regex=True) if keymatch: is_good = True for white in whitelist: if re.match(white, keymatch, re.IGNORECASE): is_good = False break if is_good: return True return False
def on_call(self, call, process): if call["api"] == "RegSetValueExA": key = self.get_argument(call, "FullName") if key and re.match(self.configkey, key): value = self.get_argument(call, "ValueName").lower() if value == "autoconfigurl": self.keybuf = self.get_argument(call, "Buffer") elif call["api"] == "NtWriteFile": path = self.get_argument(call, "HandleName") if path and re.match(self.configpath, path): buf = self.get_argument(call, "Buffer") if "user_pref" in buf and "network.proxy.autoconfig_url" in buf: tmp = buf.split("(")[1].split(")")[0].split(",")[1] self.pathbuf = tmp.strip().replace("\"", "").replace("'", "")
def run(self): urls = [] whitelist = [ "^http://crl\.microsoft\.com/.*", ] if "file" in self.results["target"]: if "PDF" in self.results["target"]["file"]["type"] or self.results["info"]["package"] == "pdf": whitelist.append("^http://.*\.adobe\.com/.*") if "network" in self.results and "http" in self.results["network"]: for req in self.results["network"]["http"]: is_whitelisted = False for white in whitelist: if re.match(white, req["uri"], re.IGNORECASE): is_whitelisted = True if not is_whitelisted and req["uri"] not in urls: urls.append(req["uri"]) for url in urls: self.data.append({"url" : url}) if urls: return True return False
def update_from(self, form, field, formfield=None, cb=False): ''' convenience method for updating fields in objects from a submitted form. allows for a callback on failure. each class has a validation_regexp dict ''' formfield = formfield if formfield else field try: value = form[formfield] if value == re.match(self.validation_regexp.get(field, '.*'), value).group(): fieldtype = type(getattr(self, field)) if fieldtype == BooleanType and type(value) == UnicodeType: setattr(self, field, value.lower() in ('true', 'yes', 'on')) else: setattr(self, field, value) else: raise AttributeError('Does not match regexp!') except KeyError: # not in form pass except AttributeError: # fails regexp! err = '"%s" is not a valid %s' % (value, field) if cb: cb(err) else: raise InvalidValue(err)
def run(self): reg_indicators = [ ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Internet\\ Explorer\\\\Toolbar\\\\.*", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Mozilla\\\\Firefox\\\\Extensions\\\\.*", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?MozillaPlugins\\\\.*", ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Google\\\\Chrome\\\\Extensions\\\\.*", ] whitelist = [ ".*\\\\SOFTWARE\\\\(Wow6432Node\\\\)?Microsoft\\\\Internet\\ Explorer\\\\Toolbar\\\\Locked$", ] found = False for indicator in reg_indicators: reg_match = self.check_write_key(pattern=indicator, regex=True, all=True) if reg_match: for match in reg_match: addit = True for white in whitelist: if not re.match(white, match, re.IGNORECASE): addit = False break if addit: self.data.append({"key": match}) found = True return found
def _unpack(self, buf): """Extract into a list irc messages of a tcp streams. @buf: tcp stream data """ try: f = cStringIO.StringIO(buf) lines = f.readlines() except Exception: log.error("Failed reading tcp stream buffer") return False logirc = False for element in lines: if not re.match("^:", element) is None: command = "([a-zA-Z]+|[0-9]{3})" params = "(\x20.+)" irc_server_msg = re.findall("(^:[\w+.{}!@|()]+\x20)" + command + params, element) if irc_server_msg: self._sc["prefix"] = convert_to_printable(irc_server_msg[0][0].strip()) self._sc["command"] = convert_to_printable(irc_server_msg[0][1].strip()) self._sc["params"] = convert_to_printable(irc_server_msg[0][2].strip()) self._sc["type"] = "server" if logirc: self._messages.append(dict(self._sc)) else: irc_client_msg = re.findall("([a-zA-Z]+\x20)(.+[\x0a\0x0d])", element) if irc_client_msg and irc_client_msg[0][0].strip() in self.__methods_client: self._cc["command"] = convert_to_printable(irc_client_msg[0][0].strip()) if self._cc["command"] in ["NICK", "USER"]: logirc = True self._cc["params"] = convert_to_printable(irc_client_msg[0][1].strip()) self._cc["type"] = "client" if logirc: self._messages.append(dict(self._cc))
def on_complete(self): # There are more, but these are the only ones I've observed execs = [ "exe", "scr", ] exts = { "doc": "Word 97-2003 document", "docx": "Word 2007+ document", "xls": "Excel 97-2003 spreadsheet", "xlsx": "Excel 2007+ spreadsheet", "ppt": "PowerPoint 97-2003 file", "pptx": "PowerPoint 2007+ file", "jpeg": "JPEG image", "jpg": "JPG image", "png": "PNG image", "gif": "GIF image", "pdf": "PDF document", "xml": "XML document", } pat = ".*[ _\-\.](?P<FakeExtension>{0})\.(?:{1})".format( "|".join(exts.keys()), "|".join(execs)) if self.get_results("target", {})["category"] == "file": check = re.match(pat, self.get_results("target", {})["file"]["name"]) if check: ext = check.group("FakeExtension") self.description = self.description.format( ext, exts[ext.lower()]) return True return False
def _add_http(self, conn, tcpdata): """Adds an HTTP flow. @param conn: TCP connection info. @param tcpdata: TCP data flow. """ if tcpdata in self.http_requests: self.http_requests[tcpdata]["count"] += 1 return True try: http = dpkt.http.Request() http.unpack(tcpdata) except dpkt.dpkt.UnpackError: pass try: entry = {"count": 1} if "host" in http.headers and re.match( "^([A-Z0-9]|[A-Z0-9][A-Z0-9\-]{0,61}[A-Z0-9])(\.([A-Z0-9]|[A-Z0-9][A-Z0-9\-]{0,61}[A-Z0-9]))+(:[0-9]{1,5})?$", http.headers["host"], re.IGNORECASE, ): entry["host"] = convert_to_printable(http.headers["host"]) else: entry["host"] = conn["dst"] if enabled_passlist: for reject in domain_passlist_re: if re.search(reject, entry["host"]): return False entry["port"] = conn["dport"] # Manually deal with cases when destination port is not the default one, # and it is not included in host header. netloc = entry["host"] if entry["port"] != 80 and ":" not in netloc: netloc += ":" + str(entry["port"]) entry["data"] = convert_to_printable(tcpdata) entry["uri"] = convert_to_printable( urlunparse(("http", netloc, http.uri, None, None, None))) entry["body"] = convert_to_printable(http.body) entry["path"] = convert_to_printable(http.uri) if "user-agent" in http.headers: entry["user-agent"] = convert_to_printable( http.headers["user-agent"]) else: entry["user-agent"] = "" entry["version"] = convert_to_printable(http.version) entry["method"] = convert_to_printable(http.method) self.http_requests[tcpdata] = entry except Exception: return False return True
def run(self, lines): _code = False _cseq = "" for l in lines: m = re.match(r"^(?P<spaces>\s*)(?P<cseq>```|~~~)\s*(?P<lang>.*)$", l) if m: yield "\r" if not _code: _code = True _cseq = m.group("cseq") if m.group("lang"): yield " #!%s\r" % m.group("lang") else: if _cseq == m.group("cseq"): _code = False _cseq = "" else: yield l continue if _code: yield " %s" % l else: yield l
def on_complete(self): whitelists = [ r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\Temporary Internet Files$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\History$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\Temporary Internet Files\\Content\.IE5\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\History\\History\.IE5\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\History\\History\.IE5\\MSHist[0-9]+\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\History\\History\.IE5\\MSHist[0-9]+\\index\.dat$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Application Data\\Microsoft\\CryptnetUrlCache\\Content\\[A-F0-9]{32}$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Application Data\\Microsoft\\CryptnetUrlCache\\Metadata\\[A-F0-9]{32}$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Cookies\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\PrivacIE\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\Local Settings\\Application Data\\Microsoft\\Feeds Cache\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\IETldCache\\$', r'^[A-Z]?:\\Documents and Settings\\[^\\]+\\IETldCache\\index.dat$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Content\.IE5\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\Cookies\\$', r'^[A-Z]?:\\Users\\[^\\]+\\Favorites\\Links\\.*', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Windows\\Temporary Internet Files\\Virtualized$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Feeds\\.*', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Feeds Cache\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Feeds Cache\\index\.dat$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IETldCache\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IETldCache\\index\.dat$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IECompatUACache\\Low$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IECompatCache\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IECompatCache\\Low$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\IECompatCache\\index.dat$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\PrivacIE\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\PrivacIE\\Low$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Roaming\\Microsoft\\Windows\\PrivacIE\\index\.dat$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Internet Explorer\\DOMStore\\.*', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Windows\\History\\History\.IE5\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Windows\\History\\History\.IE5\\MSHist[0-9]+\\$', r'^[A-Z]?:\\Users\\[^\\]+\\AppData\\Local\\Microsoft\\Windows\\History\\History\.IE5\\MSHist[0-9]+\\index\.dat$', ] saw_stealth = False target_name = None if self.is_office and "file" in self.results["target"]: target_name = self.results["target"]["file"]["name"] for hfile in self.stealth_files: addit = True for entry in whitelists: if re.match(entry, hfile, re.IGNORECASE): addit = False if self.is_office and target_name and not hfile.endswith("\\"): fname = hfile.split("\\")[-1][2:].replace("(", "_").replace(")", "_") if fname == target_name or fname in target_name: addit = False if addit: saw_stealth = True self.data.append({"file" : hfile}) return saw_stealth
def test_const_match(self): m = re2.match('abc', 'abc') self.assertIsNotNone(m) self.assertEqual(m.start(), 0) self.assertEqual(m.end(), 3) self.assertEqual(m.span(), (0, 3)) self.assertEqual(m.groups(), tuple()) self.assertEqual(m.groupdict(), {})
def on_call(self, call, process): if call["api"] == "JsEval": buf = self.get_argument(call, "Javascript") else: buf = self.get_argument(call, "Script") if re.match(".*\<object.*application\/x\-silverlight.*\<param name[ ]*=.*value[ ]*=.*\<\/object\>.*", buf, re.IGNORECASE): return True
def on_call(self, call, process): if call["api"] == "JsEval": buf = self.get_argument(call, "Javascript") else: buf = self.get_argument(call, "Script") if re.match(".*\<applet.*code[ ]*=.*archive[ ]*=.*\<\/applet\>.*", buf, re.IGNORECASE): return True
def test_group_match(self): m = re2.match('ab([cde]fg)', 'abdfghij') self.assertIsNotNone(m) self.assertEqual(m.start(), 0) self.assertEqual(m.end(), 5) self.assertEqual(m.span(), (0, 5)) self.assertEqual(m.groups(), ('dfg',)) self.assertEqual(m.groupdict(), {})
def on_call(self, call, process): if call["api"] == "JsEval": buf = self.get_argument(call, "Javascript") else: buf = self.get_argument(call, "Script") if re.match(".*allowscriptaccess[ ]*=[ ]*always.*", buf, re.IGNORECASE): return True
def validate_nickname(s): if not s or not (1 < len(s) <= 32): return False if not re.match(r"^[a-z0-9][a-z0-9-]*[a-z0-9]$", s, re.I): return False return True
def check_file_times(self, other): if ((self.createtime != 0 and self.createtime == other.createtime) or (self.lastwritetime != 0 and self.lastwritetime == other.lastwritetime) or (self.changetime != 0 and self.changetime == other.changetime)): file = other.filename.lower() if re.match(r'^[A-Z]?:\\Windows\\.*', file, re.IGNORECASE) or "\\system32\\" in file or "\\syswow64\\" in file: return other.filename return None
def run(self): whitelist = [ "^http://crl\.microsoft\.com/.*", "http://.*\.adobe\.com/.*", ] # HTTP request Features. Done like this due to for loop appending data each time instead of once so we wait to end of checks to add summary of anomalies post_noreferer = 0 post_nouseragent = 0 get_nouseragent = 0 version1 = 0 long_uri = 0 if "network" in self.results and "http" in self.results["network"]: for req in self.results["network"]["http"]: is_whitelisted = False for white in whitelist: if re.match(white, req["uri"], re.IGNORECASE): is_whitelisted = True # Check HTTP features if not is_whitelisted and req["method"] == "POST" and "Referer:" not in req["data"]: post_noreferer += 1 if not is_whitelisted and req["method"] == "POST" and "User-Agent:" not in req["data"]: post_nouseragent += 1 if not is_whitelisted and req["method"] == "GET" and "User-Agent:" not in req["data"]: get_nouseragent += 1 if not is_whitelisted and req["version"] == "1.0": version1 += 1 if post_noreferer > 0: self.data.append({"post_no_referer" : "HTTP traffic contains a POST request with no referer header" }) self.severity = 3 self.weight += 1 if post_nouseragent > 0: self.data.append({"post_no_useragent" : "HTTP traffic contains a POST request with no user-agent header" }) self.severity = 3 self.weight += 1 if get_nouseragent > 0: self.data.append({"get_no_useragent" : "HTTP traffic contains a GET request with no user-agent header" }) self.severity = 3 self.weight += 1 if version1 > 0: self.data.append({"http_version_old" : "HTTP traffic uses version 1.0" }) self.weight += 1 if self.weight: return True return False
def _is_re2_supported(): try: import re2 except ImportError: return False # re2.match doesn't work in re2 v0.2.20 installed from pypi # (it always returns None). return re2.match('foo', 'foo') is not None