def requestUrl(self, request): if request.method == DEFAULT_METHOD: return requests.request(request.url, params=request.params) else: return requests.request(request.url, method="POST", data=request.params)
def run_domain(rule): url = rule.domain + "robots.txt" response = request(url) if response and response.status_code == 200 and _ROBOTS_KEY.search(response.text): lines = response.text.splitlines() details = "\r\n".join([ lines[x] for x in range(min(len(lines),10))]) return Result(response,details)
def run_url(req, rule): print "**********webshell*******" print rule.site_type if req.params != "" or not req.url.endswith("/"): return None suffix = SITETYPES[rule.site_type] if rule.site_type is not None else None sure = [] doubt = [] response = None for line in open(os.path.join(paths.DIC, WEBSHELL_DIC_NAME)): line = line.strip() # remove \n if suffix is None: urls = ["%s%s" % (line, t) for t in SITETYPES.itervalues()] else: urls = ["%s%s" % (line, suffix)] for u in urls: url = "%s%s" % (req.url, u) print url res = request(url) if res and res.status_code == 200: text = res.text ## feature if rule.site_type is not None and rule.site_type in WEBSHELL_FEATURE: features = WEBSHELL_FEATURE[rule.site_type] if all(text.find(key) != -1 for keys in features.itervalues() for key in keys): if response is None: response = res sure.append(url) continue ## one input match = _INPUT_TYPE.findall(text) if match and len(match) == 2: if response is None: response = res doubt.append(url) continue ## match regur expression if ( ( match.group(1) for match in _IP_PATTERN.finditer(text) if all(0 <= int(x) <= 255 for x in match.group(1).split(".")) ) and _INPUT_NAMR.search(text) and _DIR_PATH.search(text) ): if response is None: response = res doubt.append(url) if response is not None: if sure: sure.insert(0, u"下列url是WebShell,请及时清理") if doubt: doubt.insert(0, u"下列url疑是为WebShell,请仔细检查!") return Result(response, sure + doubt)
def run_url(req, rule): print '**********webshell*******' print rule.site_type if req.params != '' or not req.url.endswith('/'): return None suffix = SITETYPES[rule.site_type] if rule.site_type is not None else None sure = [] doubt = [] response = None for line in open(os.path.join(paths.DIC, WEBSHELL_DIC_NAME)): line = line.strip() # remove \n if suffix is None: urls = ["%s%s" % (line, t) for t in SITETYPES.itervalues()] else: urls = ["%s%s" % (line, suffix)] for u in urls: url = "%s%s" % (req.url, u) print url res = request(url) if res and res.status_code == 200: text = res.text ## feature if rule.site_type is not None and rule.site_type in WEBSHELL_FEATURE: features = WEBSHELL_FEATURE[rule.site_type] if all( text.find(key) != -1 for keys in features.itervalues() for key in keys): if response is None: response = res sure.append(url) continue ## one input match = _INPUT_TYPE.findall(text) if match and len(match) == 2: if response is None: response = res doubt.append(url) continue ## match regur expression if (match.group(1) for match in _IP_PATTERN.finditer(text) if all(0<=int(x)<=255 for x in match.group(1).split('.'))) \ and _INPUT_NAMR.search(text) \ and _DIR_PATH.search(text) : if response is None: response = res doubt.append(url) if response is not None: if sure: sure.insert(0, u"下列url是WebShell,请及时清理") if doubt: doubt.insert(0, u"下列url疑是为WebShell,请仔细检查!") return Result(response, sure + doubt)
def destReachable(dest=None): if not dest: dest = conf.url response = request(dest, timeout=conf.connect_timeout) if response is None: set_unreachable_flag(conf.taskid) raise DestinationUnReachable(dest) else: conf.site_type = sitetype_check(response)
def destReachable(dest=None): if not dest: dest = conf.url response = request(dest,timeout=conf.connect_timeout) if response is None: set_unreachable_flag(conf.taskid) raise DestinationUnReachable(dest) else: conf.site_type = sitetype_check(response)
def run_domain(rule): admin = ('phpmyadmin','phpMyAdmin','db') keys = ('<form method="post" action="index.php" name="login_form" target="_top">', '<input type="text" name="pma_username" value="" size="24" class="textfield"') for p in admin: response = request(rule.domain + p) if response and response.status_code == 200 and all((response.text.find(k)!=-1 for k in keys)): match = _PHPMYADMIN_VERSION.search(response.text) details = u"phpMyAdmin版本:" + match.group(1).strip() if match and match.group(1).strip() else '' return Result(response,details)
def parseRobots(self): """ parse robots protocol,both allow and disallow entry for example: http://www.gaoloumi.com/robots.txt """ self.task.update_robots_flag('start') robotsUrl = self.domain + "robots.txt" try: response = requests.request(robotsUrl) if not response: return lines = response.text.splitlines() for line in lines: match = ROBOTS_ALLOW_PATH.search(line) path = match.group('path') if match else '/' if path != '/': url = urljoin(self.domain,path) self.addRequest(Request.fromUrl(url,robotsUrl)) finally: self.task.update_robots_flag('finish')
def parseRobots(self): """ parse robots protocol,both allow and disallow entry for example: http://www.gaoloumi.com/robots.txt """ self.task.update_robots_flag('start') robotsUrl = self.domain + "robots.txt" try: response = requests.request(robotsUrl) if not response: return lines = response.text.splitlines() for line in lines: match = ROBOTS_ALLOW_PATH.search(line) path = match.group('path') if match else '/' if path != '/': url = urljoin(self.domain, path) self.addRequest(Request.fromUrl(url, robotsUrl)) finally: self.task.update_robots_flag('finish')
def parseSitemap(self): """ parse sitemap.xml for example: http://www.aouu.com/sitemap.xml """ self.task.update_sitemap_parsed('start') sitemapUrl = self.domain + "sitemap.xml" try: response = requests.request(sitemapUrl) if not response: return lines = response.iter_lines() #sitemap.xml may very big for line in lines: match = SITEMAP_URL.search(line) if match: url = match.group('url') if self.isOrigin(url) and url not in self.urlSet: self.urlSet.add(url) self.addRequest(Request.fromUrl(url,sitemapUrl)) finally: self.task.update_sitemap_parsed('finish')
def parseSitemap(self): """ parse sitemap.xml for example: http://www.aouu.com/sitemap.xml """ self.task.update_sitemap_parsed('start') sitemapUrl = self.domain + "sitemap.xml" try: response = requests.request(sitemapUrl) if not response: return lines = response.iter_lines() #sitemap.xml may very big for line in lines: match = SITEMAP_URL.search(line) if match: url = match.group('url') if self.isOrigin(url) and url not in self.urlSet: self.urlSet.add(url) self.addRequest(Request.fromUrl(url, sitemapUrl)) finally: self.task.update_sitemap_parsed('finish')
def requestUrl(self,request): if request.method == DEFAULT_METHOD: return requests.request(request.url,params=request.params) else: return requests.request(request.url,method="POST",data=request.params)