def requestUrl(self, request):
if request.method == DEFAULT_METHOD:
return requests.request(request.url, params=request.params)
else:
return requests.request(request.url,
method="POST",
data=request.params)
def run_domain(rule):
url = rule.domain + "robots.txt"
response = request(url)
if response and response.status_code == 200 and _ROBOTS_KEY.search(response.text):
lines = response.text.splitlines()
details = "\r\n".join([ lines[x] for x in range(min(len(lines),10))])
return Result(response,details)
def run_url(req, rule):
print "**********webshell*******"
print rule.site_type
if req.params != "" or not req.url.endswith("/"):
return None
suffix = SITETYPES[rule.site_type] if rule.site_type is not None else None
sure = []
doubt = []
response = None
for line in open(os.path.join(paths.DIC, WEBSHELL_DIC_NAME)):
line = line.strip() # remove \n
if suffix is None:
urls = ["%s%s" % (line, t) for t in SITETYPES.itervalues()]
else:
urls = ["%s%s" % (line, suffix)]
for u in urls:
url = "%s%s" % (req.url, u)
print url
res = request(url)
if res and res.status_code == 200:
text = res.text
## feature
if rule.site_type is not None and rule.site_type in WEBSHELL_FEATURE:
features = WEBSHELL_FEATURE[rule.site_type]
if all(text.find(key) != -1 for keys in features.itervalues() for key in keys):
if response is None:
response = res
sure.append(url)
continue
## one input
match = _INPUT_TYPE.findall(text)
if match and len(match) == 2:
if response is None:
response = res
doubt.append(url)
continue
## match regur expression
if (
(
match.group(1)
for match in _IP_PATTERN.finditer(text)
if all(0 <= int(x) <= 255 for x in match.group(1).split("."))
)
and _INPUT_NAMR.search(text)
and _DIR_PATH.search(text)
):
if response is None:
response = res
doubt.append(url)
if response is not None:
if sure:
sure.insert(0, u"下列url是WebShell，请及时清理")
if doubt:
doubt.insert(0, u"下列url疑是为WebShell，请仔细检查!")
return Result(response, sure + doubt)
def run_url(req, rule):
print '**********webshell*******'
print rule.site_type
if req.params != '' or not req.url.endswith('/'):
return None
suffix = SITETYPES[rule.site_type] if rule.site_type is not None else None
sure = []
doubt = []
response = None
for line in open(os.path.join(paths.DIC, WEBSHELL_DIC_NAME)):
line = line.strip() # remove \n
if suffix is None:
urls = ["%s%s" % (line, t) for t in SITETYPES.itervalues()]
else:
urls = ["%s%s" % (line, suffix)]
for u in urls:
url = "%s%s" % (req.url, u)
print url
res = request(url)
if res and res.status_code == 200:
text = res.text
## feature
if rule.site_type is not None and rule.site_type in WEBSHELL_FEATURE:
features = WEBSHELL_FEATURE[rule.site_type]
if all(
text.find(key) != -1
for keys in features.itervalues() for key in keys):
if response is None:
response = res
sure.append(url)
continue
## one input
match = _INPUT_TYPE.findall(text)
if match and len(match) == 2:
if response is None:
response = res
doubt.append(url)
continue
## match regur expression
if (match.group(1) for match in _IP_PATTERN.finditer(text) if all(0<=int(x)<=255 for x in match.group(1).split('.'))) \
and _INPUT_NAMR.search(text) \
and _DIR_PATH.search(text) :
if response is None:
response = res
doubt.append(url)
if response is not None:
if sure:
sure.insert(0, u"下列url是WebShell，请及时清理")
if doubt:
doubt.insert(0, u"下列url疑是为WebShell，请仔细检查!")
return Result(response, sure + doubt)
def destReachable(dest=None):
if not dest:
dest = conf.url
response = request(dest, timeout=conf.connect_timeout)
if response is None:
set_unreachable_flag(conf.taskid)
raise DestinationUnReachable(dest)
else:
conf.site_type = sitetype_check(response)
def destReachable(dest=None):
if not dest:
dest = conf.url
response = request(dest,timeout=conf.connect_timeout)
if response is None:
set_unreachable_flag(conf.taskid)
raise DestinationUnReachable(dest)
else:
conf.site_type = sitetype_check(response)
def run_domain(rule):
admin = ('phpmyadmin','phpMyAdmin','db')
keys = ('<form method="post" action="index.php" name="login_form" target="_top">',
'<input type="text" name="pma_username" value="" size="24" class="textfield"')
for p in admin:
response = request(rule.domain + p)
if response and response.status_code == 200 and all((response.text.find(k)!=-1 for k in keys)):
match = _PHPMYADMIN_VERSION.search(response.text)
details = u"phpMyAdmin版本:" + match.group(1).strip() if match and match.group(1).strip() else ''
return Result(response,details)
def parseRobots(self):
"""
parse robots protocol,both allow and disallow entry
for example: http://www.gaoloumi.com/robots.txt
"""
self.task.update_robots_flag('start')
robotsUrl = self.domain + "robots.txt"
try:
response = requests.request(robotsUrl)
if not response:
return
lines = response.text.splitlines()
for line in lines:
match = ROBOTS_ALLOW_PATH.search(line)
path = match.group('path') if match else '/'
if path != '/':
url = urljoin(self.domain,path)
self.addRequest(Request.fromUrl(url,robotsUrl))
finally:
self.task.update_robots_flag('finish')
def parseRobots(self):
"""
parse robots protocol,both allow and disallow entry
for example: http://www.gaoloumi.com/robots.txt
"""
self.task.update_robots_flag('start')
robotsUrl = self.domain + "robots.txt"
try:
response = requests.request(robotsUrl)
if not response:
return
lines = response.text.splitlines()
for line in lines:
match = ROBOTS_ALLOW_PATH.search(line)
path = match.group('path') if match else '/'
if path != '/':
url = urljoin(self.domain, path)
self.addRequest(Request.fromUrl(url, robotsUrl))
finally:
self.task.update_robots_flag('finish')
def parseSitemap(self):
"""
parse sitemap.xml
for example: http://www.aouu.com/sitemap.xml
"""
self.task.update_sitemap_parsed('start')
sitemapUrl = self.domain + "sitemap.xml"
try:
response = requests.request(sitemapUrl)
if not response:
return
lines = response.iter_lines() #sitemap.xml may very big
for line in lines:
match = SITEMAP_URL.search(line)
if match:
url = match.group('url')
if self.isOrigin(url) and url not in self.urlSet:
self.urlSet.add(url)
self.addRequest(Request.fromUrl(url,sitemapUrl))
finally:
self.task.update_sitemap_parsed('finish')
def parseSitemap(self):
"""
parse sitemap.xml
for example: http://www.aouu.com/sitemap.xml
"""
self.task.update_sitemap_parsed('start')
sitemapUrl = self.domain + "sitemap.xml"
try:
response = requests.request(sitemapUrl)
if not response:
return
lines = response.iter_lines() #sitemap.xml may very big
for line in lines:
match = SITEMAP_URL.search(line)
if match:
url = match.group('url')
if self.isOrigin(url) and url not in self.urlSet:
self.urlSet.add(url)
self.addRequest(Request.fromUrl(url, sitemapUrl))
finally:
self.task.update_sitemap_parsed('finish')
def requestUrl(self,request):
if request.method == DEFAULT_METHOD:
return requests.request(request.url,params=request.params)
else:
return requests.request(request.url,method="POST",data=request.params)