def init(self):
try:
if self.action == 'restart':
logger.debug('restart to scan weak pwd')
self.dao.deleteData('weak_pwd_result', {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId
})
self.dao.updateData('host_infos', {'weak_pwd_scan_state': 0}, {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId
})
self.dao.updateData('task', {'weak_pwd_scan_state': 0},
{'id': self.taskId})
#end if
self.taskCnf = self.dao.getTaskData(self.taskId)
if self.taskCnf == False:
return False
#end if
self.assetTaskId = self.taskCnf['asset_task_id']
self.thread = self.taskCnf['weak_pwd_scan_thread']
self.timeout = self.taskCnf['weak_pwd_scan_timeout']
self.policy = self.taskCnf['weak_pwd_scan_policy'].split(',')
return True
except Exception, e:
logger.error(e)
return False
def run(self):
try:
logger.debug('Host scan is start')
if self.init() == False:
return
#end if
logger.debug('Host scan init is success')
if self.taskCnf['host_scan_enable'] == 0:
self.finish()
return
#end if
logger.debug('Host scan is enable')
if self.taskCnf['host_scan_enable'] == 1 and self.taskCnf[
'host_scan_state'] == 1:
self.finish()
return
#end if
logger.debug('Host scan is not finished')
res = self.dao.getData(
'host_infos', {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId,
'host_scan_state': 0
})
if not res or len(res) <= 0:
self.finish()
return
#end if
logger.debug('find host_scan target to scan')
for row in res:
self.target.put(row['ip'])
#end for
threadList = []
for i in range(self.thread):
t = HostScanThread(self.taskCnf, self.target)
threadList.append(t)
#end for
for t in threadList:
t.start()
#end for
for t in threadList:
t.join()
#end for
self.finish()
except Exception, e:
logger.error(e)
def run(self):
try:
cmd = "%s -sS -p 1-9999 -V -i %s %s" % (NASL_EXE, self.vulIds,
self.ip)
logger.debug(cmd)
popenData = popen(cmd)
res = self.handleResult(popenData)
self.updateHostResult(res)
except Exception, e:
logger.error(e)
def init(self):
try:
if self.action == 'restart':
logger.debug('restart to scan port')
self.dao.deleteData('host_ports', {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId
})
self.dao.updateData('host_infos', {'port_scan_state': 0}, {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId
})
self.dao.updateData('task', {'port_scan_state': 0},
{'id': self.taskId})
#end if
self.taskCnf = self.dao.getTaskData(self.taskId)
if self.taskCnf == False:
return False
#end if
logger.debug('get task config success')
self.assetTaskId = self.taskCnf['asset_task_id']
self.thread = self.taskCnf['port_scan_thread']
self.timeout = self.taskCnf['port_scan_timeout']
self.policy = self.taskCnf['port_scan_policy']
res = self.dao.getRowData('port_scan_policy', {'id': self.policy})
if res and len(res) > 0:
self.ports = res['ports']
logger.debug('get port scan policy success')
#end if
if self.taskCnf['port_scan_enable'] == 1 and self.taskCnf[
'port_scan_state'] == 0:
logger.debug('port scan init success')
return True
#end if
logger.debug('port scan is finished')
self.dao.updateData('host_infos', {'port_scan_state': 1}, {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId,
'state': 1
})
return False
except Exception, e:
logger.error(e)
return False
def __init__(self, taskId, assetTaskId, taskCnf, threadLock):
logger.debug("__init__ scansite")
try:
threading.Thread.__init__(self)
self.module = self.__class__.__name__
self.taskId = taskId
self.assetTaskId = assetTaskId
self.taskCnf = taskCnf
self.threadLock = threadLock
self.threadName = threading.currentThread().getName()
self.dao = MysqlDao()
self.count = 0
except Exception, e:
logger.exception(e)
def main(self):
try:
# add params domain use for init host_scan of the header by mcj
http = HttpRequest({
'domain': self.domain,
'timeout': self.webTimeout,
'follow_redirects': True,
'cookie': self.cookie
})
logger.debug("from plugins.%s import *" % (self.script))
exec("from plugins.%s import *" % (self.script))
#临时计数器
temp_num = 0
#开始扫描
while True:
try:
if self.urlQueue.qsize() < 1:
break
list = []
try:
item = self.urlQueue.get_nowait()
except Exception, e2:
break
url = item['url']
if item['method'] == 'get' and item['params'] != '':
url = "%s?%s" % (url, item['params'])
if self.checkExcludeUrl(self.excludeUrl, url):
continue
list = run_url(http, self.scanCnf, item)
if list and len(list) > 0:
temp_num += len(list)
#对于邮件信息泄露和HTML注释信息泄露做了部分特殊处理
if self.script in [
'HtmlSourceLeakScript', 'EmailDiscloseScript'
] and temp_num > 10:
break
self.threadLock.acquire()
self.updateResult(list)
self.threadLock.release()
except Exception, e1:
logger.error(e1)
def start(self):
try:
if self.script == "":
return
# add params domain use for init host_scan of the header by mcj
http = HttpRequest({
'domain': self.domain,
'timeout': self.webTimeout,
'follow_redirects': True
})
logger.debug("from plugins.%s import *" % (self.script))
exec("from plugins.%s import *" % (self.script))
#开始扫描
list = run_domain(http, self.scanCnf)
if list and len(list) > 0:
self.updateResult(list)
except Exception, e:
logger.error(e)
def init(self):
try:
self.taskCnf = self.dao.getTaskData(self.taskId)
if self.taskCnf == False:
return False
self.assetTaskId = self.taskCnf['asset_task_id']
self.taskCnf['vulDict'] = {}
self.taskCnf['vulList'] = []
vulList = self.dao.getWebVulByPolicy(
self.taskCnf['web_scan_policy'])
for row in vulList:
vulId = str(row['vul_id'])
self.taskCnf['vulDict'][vulId] = row
self.taskCnf['vulList'].append(vulId)
return True
except Exception, e:
logger.debug(e)
return False
def run(self):
try:
logger.debug('start to scan site')
i = 0
while True:
i += 1
logger.debug("try to find new site, i: %d" % (i))
try:
self.threadLock.acquire()
checkSiteUnscand = self.checkSiteUnScaned()
self.threadLock.release()
if checkSiteUnscand:
break
siteId = None
try:
siteId = siteQueue.get(True, 30)
except Exception, ge:
pass
#开始这个域名的扫描
if siteId and self.checkSiteId(siteId):
self.count = 0
logger.debug("find new site to scan, siteId: %s" %
(siteId))
self.scanSiteMain(str(siteId))
self.checkExceptionSite()
except Exception, e1:
logger.error(e1)
self.count += 1
def scanMain(self):
try:
logger.debug('weak pwd scan main is start')
while True:
if self.target.qsize() < 1:
break
#end if
ip = self.target.get_nowait()
if ip and ip != '':
logger.debug("weak pwd scan man is scaning ip: %s" % (ip))
scanList = self.checkIpPort(ip)
if scanList and len(scanList) > 0:
for line in scanList:
self.crack(ip, line)
#end for
#end if
#更新主机状态
self.updateHostState(ip)
#end if
#end while
except Exception, e:
logger.error(e)
def run(self):
try:
while True:
if self.queue.qsize() < 1:
print "prescan thread exit"
break
#end if
ip = self.queue.get_nowait()
if not ip or ip == "":
print "prescan thread exit"
break
#end if
if self.checkIp(ip):
self.dao.updateData('host_infos', {'state':1}, {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip})
else:
logger.debug("ip is not alive: %s" % (ip))
self.dao.updateData('host_infos', {'state':2,'port_scan_state':1,'host_scan_state':1,'web_scan_state':1,'weak_pwd_scan_state':1}, {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip})
#end if
#end while
except Exception,e:
logger.error(e)
def run(self):
try:
logger.debug('start to search site')
# 获取上一次未扫描完成的域名
logger.debug('taskId %s ' % self.taskId)
siteIds = self.dao.getUnscandSite(self.taskId, self.assetTaskId)
logger.debug(siteIds)
for siteId in siteIds:
logger.debug('siteQueue put %s ' % siteId)
siteQueue.put(siteId)
if self.taskCnf['web_search_site_state'] == 0:
target = self.taskCnf['target'].encode('utf8')
if target == '':
target = []
else:
target = json.read(target)
ipList = []
logger.debug('target: ' + jsonSys.dumps(target))
for row in target:
try:
siteObj = {
'scheme': 'http',
'domain': '',
'path': '/',
'ip': '',
'title': '',
'policy': '1',
'include_url': '',
'exclude_url': '',
'cookie': '',
'sub_domain_scan': 0,
'ip_domain_scan': 0
}
if row.has_key('scheme'):
siteObj['scheme'] = row['scheme']
if row.has_key('domain') == False:
logger.error("can not get domain")
continue
siteObj['domain'] = row['domain']
if row.has_key('path'):
siteObj['path'] = row['path']
if row.has_key('ip'):
# 支持配置自定义IP指向
ip = row['ip']
else:
ip = domainToip(siteObj['domain'])
if ip == False:
logger.error("can not get ip, domain: %s" %
(siteObj['domain']))
continue
siteObj['ip'] = ip
if row.has_key('title'):
siteObj['title'] = row['title']
if row.has_key('policy'):
siteObj['policy'] = row['policy']
else:
siteObj['policy'] = 1
if row.has_key('include_url'):
siteObj['include_url'] = json.write(
row['include_url'])
if row.has_key('exclude_url'):
siteObj['exclude_url'] = json.write(
row['exclude_url'])
if row.has_key('cookie'):
siteObj['cookie'] = row['cookie']
# 是否扫描二级域名
if row.has_key('sub_domain_scan'):
siteObj['sub_domain_scan'] = row['sub_domain_scan']
# 该IP的其他域名扫描
if row.has_key('ip_domain_scan'):
siteObj['ip_domain_scan'] = row['ip_domain_scan']
if siteObj['ip_domain_scan'] == 1:
# 根据IP获取IP下的域名
if ip not in ipList:
otherSites = self.searchOtherSiteInIp(ip)
target.extend(otherSites)
ipList.append(ip)
self.updateTaskSites(siteObj)
except Exception, ee:
logger.error(ee)
self.finish()
logger.debug('end to search site')
def crack(self, ip, type):
try:
logger.debug('start to crack ip: %s, vulName: %s' % (ip, type))
dicFile = "dict/%s.dic" % (type)
f = open(dicFile, "r")
lines = f.readlines()
startTime = time.time()
if type == "telnet":
telnet_check = Telnet_login(ip)
#end if
for line in lines:
try:
if int(time.time() - startTime) > self.timeout:
break
#end if
if type == "vnc":
username = ""
password = line.strip()
password = password.replace("\\", "\\\\")
password = password.replace("`", "\\`")
password = password.replace("\"", "\\\"")
else:
up = line.split(":", 1)
if len(up) < 2:
continue
#end if
username = up[0].strip()
password = up[1].strip()
username = username.replace("\\", "\\\\")
username = username.replace("`", "\\`")
username = username.replace("\"", "\\\"")
password = password.replace("\\", "\\\\")
password = password.replace("`", "\\`")
password = password.replace("\"", "\\\"")
#end if
cmd = ""
if type == "vnc":
if password == "{NULL}":
cmd = "hydra -t 1 -sid=%s -p \"\" %s %s" % (
self.taskId, ip, type)
else:
cmd = "hydra -t 1 -sid=%s -p \"%s\" %s %s" % (
self.taskId, password, ip, type)
#end if
elif type == "oracle":
if password == "{NULL}":
cmd = "hydra -t 1 -sid=%s -l \"%s\" -e n %s %s ORCL" % (
self.taskId, username, ip, type)
else:
cmd = "hydra -t 1 -sid=%s -l \"%s\" -p \"%s\" %s %s ORCL" % (
self.taskId, username, password, ip, type)
#end if
else:
if password == "{NULL}":
cmd = "hydra -t 1 -sid=%s -l \"%s\" -e n %s %s" % (
self.taskId, username, ip, type)
else:
cmd = "hydra -t 1 -sid=%s -l \"%s\" -p \"%s\" %s %s" % (
self.taskId, username, password, ip, type)
#end if
#end if
logger.debug(cmd)
(output, exitstatus) = run(cmd,
withexitstatus=1,
timeout=20)
logger.debug(output)
#m = self.compiledRule.findall(output)
#print output
#print m
#if len(m) == 1:
if output.find('1 valid password found') > 0:
#v = m[0]
#username = v[3].strip()
#password = v[4].strip()
if type == "telnet":
telnet_passwd = '' if len(
password) == 0 else password
if not telnet_check.check(username, telnet_passwd):
continue
#end if
#end if
if len(password) == 0:
self.updateResult(ip, type.upper(), username,
"空密码")
if type.upper() == "FTP":
self.updateResult(ip, type.upper(), "ftp",
"ftp")
#end if
else:
self.updateResult(ip, type.upper(), username,
password)
#end if
return
#end if
except Exception, e1:
logger.error(e)
continue
#end try
#end for
except Exception, e:
logger.error(e)
def scanSiteMain(self, siteId):
try:
logger.debug("start to scan site, siteId: %s" % (siteId))
if siteId == None:
return False
dao = MysqlDao()
siteObj = dao.getSiteData(siteId)
if siteObj == None:
logger.error("start to get site config exception, siteId: %s" %
(siteId))
return False
#scheme
scheme = siteObj['scheme'].encode('utf8')
#ip address
ip = siteObj['ip'].encode('utf8')
#site domain
domain = siteObj['domain'].encode('utf8')
#site scan state
state = siteObj['state']
#site path
path = siteObj['path'].encode('utf8')
#site title
title = siteObj['title'].encode('utf8')
#site type
siteType = siteObj['site_type'].encode('utf8')
#site cookie
cookie = siteObj['cookie'].encode('utf8')
#site include url
includeUrl = siteObj['include_url'].encode('utf8')
if includeUrl == '':
includeUrl = []
else:
includeUrl = json.read(includeUrl)
#site exclude url
excludeUrl = siteObj['exclude_url'].encode('utf8')
if excludeUrl == '':
excludeUrl = []
else:
excludeUrl = json.read(excludeUrl)
#scan progress
progress = siteObj['progress'].encode('utf8')
#site scan policy
policy = siteObj['policy']
logger.debug("scanSiteMain siteId: %s" % (siteId))
if state == 1:
self.finishSiteScan(siteId, ip)
return True
#在DNS配置文件中加入这个域名的DNS信息
self.threadLock.acquire()
self.updateHosts(ip, domain, self.taskId, siteId, 'add')
self.threadLock.release()
flag = res = content = checkOk = None
target = []
logger.debug("scanSiteMain siteId: %s preSiteScan before" %
(siteId))
target.append("%s://%s%s" % (scheme, domain, path))
for url in target:
flag, res, content = self.PreSiteScan(url)
if not flag:
continue
else:
if self.checkSiteWorkMode(res, title) == False:
continue
else:
checkOk = 1
break
if not checkOk:
self.updateSiteException("网站无法访问", siteId, ip)
return
else:
siteCode = self.getSiteCode(content)
if title == "" and res and res.has_key(
'status') and res['status'] == '200':
title = self.updateSiteTitle(content, siteId)
if siteType == "":
siteType = self.updateSiteType(res, siteId)
if siteCode == "":
siteCode = self.getSiteCode(content)
if self.taskCnf['web_scan_timeout']:
socket.setdefaulttimeout(self.taskCnf['web_scan_timeout'])
siteDb = {'state': 0, 'exception': ''}
if siteObj['start_time'] is None or siteObj[
'start_time'] == '0000-00-00 00:00:00':
siteDb['start_time'] = time.strftime("%Y-%m-%d %X",
time.localtime())
if siteObj['progress'] == '':
siteDb['progress'] = '0'
self.dao.updateData('sites', siteDb, {'id': siteId})
logger.debug("scanSiteMain siteId: %s policy before" % (siteId))
###############################
#policy:
# 1:快速扫描，只扫描指定的域名
# 2:完全扫描，扫描指定的域名，并且扫描二级域名
# 3:扫描指定目录及子目录
# 4:扫描指定的URL，这个情况下，不需要爬虫
# 5:通过域名反查得到的域名
# 6:登陆型扫描
###############################
if self.taskCnf['spider_enable'] == 1 and siteObj[
'spider_state'] == 0:
logger.debug('spider is start')
progress = '0'
logger.debug("scanSiteMain siteId: %s cleandata before" %
(siteId))
self.dao.deleteData('web_result', {'site_id': siteId})
self.dao.deleteData('web_result_data', {'site_id': siteId})
self.dao.deleteData('spider_url', {'site_id': siteId})
#开启爬虫，当扫描指定的URL时，不需要爬虫
if siteObj['policy'] != 4:
spiderCnf = {}
spiderCnf['taskId'] = self.taskId
spiderCnf['assetTaskId'] = self.assetTaskId
spiderCnf['siteId'] = siteId
spiderCnf['spiderUrlCount'] = self.taskCnf[
'spider_url_count']
spiderCnf['webScanTime'] = self.taskCnf['web_scan_timeout']
spiderCnf['policy'] = siteObj['policy']
spiderCnf['scheme'] = siteObj['scheme'].encode('utf8')
spiderCnf['domain'] = domain
spiderCnf['path'] = path
spiderCnf['maxTimeCount'] = 30
spiderCnf['webScanTimeout'] = self.taskCnf[
'web_scan_timeout']
spiderCnf['endTime'] = time.time() + 1800
spiderCnf['maxnum'] = self.taskCnf['spider_url_count']
spiderCnf['title'] = title
spiderCnf['ip'] = ip
spiderCnf['cookie'] = cookie
spiderCnf['webSearchSiteState'] = self.taskCnf[
'web_search_site_state']
spiderCnf['webSearchSiteTimeout'] = self.taskCnf[
'web_search_site_timeout']
spiderCnf['includeUrl'] = includeUrl
spiderCnf['excludeUrl'] = excludeUrl
spiderCnf['downloadDir'] = './log/'
logger.debug("scanSiteMain siteId: %s startSpider before" %
(siteId))
if self.taskCnf['spider_type'] == 2:
import Spider2 as Spider
else:
import Spider
logger.debug("spiderCnf start")
logger.debug(spiderCnf)
logger.debug("spiderCnf end")
spider = Spider.Spider(spiderCnf)
spider.start()
logger.debug('spider is end')
self.dao.updateData('sites', {'spider_state': 1}, {'id': siteId})
####################################### 至此蜘蛛结束，后面是扫描的工作 #############################################
return True
siteCnf = dao.getSiteData(siteId)
domain = siteCnf['domain'].encode('utf8')
path = siteCnf['path'].encode('utf8')
title = siteCnf['title']
if title != '':
title = title.encode('utf8')
#检测网站的状态，有的网站访问后直接访问500或者其他的情况。
if self.checkSiteWorkMode({}, title) == False:
self.finishSiteScan(siteId, ip)
return
logger.debug('get site scan config')
scanCnf = {}
scanCnf['taskId'] = self.taskId
scanCnf['assetTaskId'] = self.assetTaskId
scanCnf['siteId'] = siteId
scanCnf['maxThread'] = 10
scanCnf['scriptThread'] = 10
scanCnf['webTimeout'] = self.taskCnf['web_scan_timeout']
scanCnf['ip'] = ip
scanCnf['scheme'] = scheme
scanCnf['domain'] = domain
scanCnf['path'] = path
scanCnf['cookie'] = cookie
scanCnf['errorCount'] = 0
scanCnf['errorLenDict'] = {}
scanCnf['siteType'] = siteType
scanCnf['maxTimeoutCount'] = 20
scanCnf['siteCode'] = siteCode
scanCnf['cookie'] = cookie
scanCnf['len404'] = []
scanCnf['isForce'] = 0
scanCnf['excludeUrl'] = excludeUrl
scanCnf['threadLock'] = threading.Lock()
scanCnf['isstart'] = '1'
#判断该域名扫描进度，加载未扫描的漏洞ID
logger.debug('load unscaned script start')
scanVulList = []
progress = progress.split('|')
for vulId in self.taskCnf['vulList']:
if vulId not in progress:
scanVulList.append(vulId)
logger.debug('script scan is start')
if len(scanVulList) > 0:
urlList = []
if policy == 4:
for url in includeUrl:
if url in excludeUrl:
continue
t = r.split('?')
url = t[0]
params = ''
if len(t) > 1:
params = t[1]
urlList.append({
'url': url,
'params': params,
'method': 'get'
})
else:
res = self.dao.getUrlList(siteId)
for r in res:
url = r['url'].encode('utf8')
if nonascii(url): url = safeUrlString(url)
urlList.append({
'url': url,
'params': r['params'].encode('utf8'),
'method': r['method'].encode('utf8'),
'refer': r['refer'].encode('utf8')
})
for vulId in scanVulList:
vulId = vulId.replace(" ", "")
if vulId == "":
continue
progress.append(vulId)
self.dao.updateData('sites',
{'progress': '|'.join(progress)},
{'id': siteId})
self.dao.deleteData('web_result', {
'vul_id': vulId,
'site_id': siteId
})
scanCnf['vulId'] = vulId
scanCnf['vulName'] = self.taskCnf['vulDict'][vulId][
'vul_name']
scanCnf['level'] = self.taskCnf['vulDict'][vulId][
'level'].encode('utf8')
scanCnf['scanType'] = self.taskCnf['vulDict'][vulId][
'scan_type']
scanCnf['script'] = self.taskCnf['vulDict'][vulId][
'script']
scanCnf['status'] = '0'
scanCnf['endTime'] = time.time() + 1800
scanCnf['timeoutCount'] = 0
#测试爬虫爬出来的路径
if scanCnf['scanType'] == 1:
scanCnf['queue'] = Queue()
for r in urlList:
scanCnf['queue'].put(r)
scanUrlScript = ScanScriptForUrl(scanCnf)
scanUrlScript.start()
#如果只测试指定的URL则不需要运行测试域名和测试漏洞库
if policy != 4:
#测试域名
if scanCnf['scanType'] == 2:
scanDomainScript = ScanScriptForDomain(scanCnf)
scanDomainScript.start()
urlList = []
#结束扫描
self.finishSiteScan(siteId, ip)
self.threadLock.acquire()
self.updateHosts(ip, domain, self.taskId, siteId, 'remove')
self.threadLock.release()
return None
except Exception, e:
logger.exception(e)
return siteId
def init(self):
logger.debug("init scansite")
try:
pass
except Exception, e:
logger.exception(e)
def updateTaskSites(self, siteObj):
try:
logger.debug('start to add new site')
try:
scheme, domain, path = getRedirect(
"%s://%s%s" %
(siteObj['scheme'], siteObj['domain'], siteObj['path']))
except Exception, e1:
scheme = siteObj['scheme']
domain = siteObj['domain']
path = siteObj['path']
pass
if scheme != 'http' and scheme != 'https':
siteObj['scheme'] = 'http'
else:
siteObj['scheme'] = scheme
if not path or path == '':
siteObj['path'] = path
siteExistObj = {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId,
'scheme': siteObj['scheme'],
'domain': siteObj['domain'],
'path': siteObj['path']
}
if self.dao.getDataCount('sites', siteExistObj) > 0:
return True
siteObj['task_id'] = self.taskId
siteObj['asset_task_id'] = self.assetTaskId
siteObj['progress'] = ''
siteId = self.dao.insertData('sites', siteObj)
if siteId == 0:
logger.error('add new site to db exception')
return False
siteQueue.put(str(siteId))
####### use the new spider add by mcj
self.dao.updateData('spider_url', {'site_id': siteId},
{'task_id': self.taskId})
self.dao.updateData('spider_url_other', {'site_id': siteId},
{'task_id': self.taskId})
#############
ipExistObj = {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId,
'ip': siteObj['ip']
}
if self.dao.getDataCount('host_infos', ipExistObj) < 1:
currentTime = time.strftime("%Y-%m-%d %X", time.localtime())
hostObj = {
'task_id': self.taskId,
'asset_task_id': self.assetTaskId,
'ip': siteObj['ip'],
'start_time': currentTime
}
# mcj:insertData() takes exactly 3 arguments (2 given) add argument:host_infos
self.dao.insertData('host_infos', hostObj)
taskObj = {
'host_scan_state': 0,
'weak_pwd_scan_state': 0,
'port_scan_state': 0
}
self.dao.updateData('task', taskObj, {'id': self.taskId})
logger.debug("try to find new site, i: %d" % (i))
try:
self.threadLock.acquire()
checkSiteUnscand = self.checkSiteUnScaned()
self.threadLock.release()
if checkSiteUnscand:
break
siteId = None
try:
siteId = siteQueue.get(True, 30)
except Exception, ge:
pass
#开始这个域名的扫描
if siteId and self.checkSiteId(siteId):
self.count = 0
logger.debug("find new site to scan, siteId: %s" %
(siteId))
self.scanSiteMain(str(siteId))
self.checkExceptionSite()
except Exception, e1:
logger.error(e1)
self.count += 1
if self.count > 2:
break
logger.debug('end to scan site')
except Exception, e:
logger.debug(e)
def run(self):
try:
logger.debug('clear last scan')
if self.action == 'restart':
if self.clean() == False:
logger.error('清空上一次扫描失败')
return False
logger.debug('init scan config')
#初始化扫描任务
if self.init() == False:
logger.error('初始化扫描信息失败')
return False
logger.debug('check init state')
if self.taskCnf['init_state'] == 0:
logger.error('初始化失败')
return False
logger.debug('check if need to scan web vul')
if self.taskCnf['web_scan_enable'] == 0:
logger.debug('不需要扫描Web漏洞')
self.finish()
return True
logger.debug('check web scan is finished')
if self.taskCnf['web_scan_state'] == 1:
logger.debug('Web扫描已完成')
return True
#调用获取域名的线程和扫描域名的线程
threadList = []
threadList.append(SearchSite(self.taskId, self.taskCnf))
for i in range(self.taskCnf['web_scan_thread']):
threadList.append(
ScanSite(self.taskId, self.assetTaskId, self.taskCnf,
self.threadLock))
for t in threadList:
t.start()
for t in threadList:
t.join()
self.finish()
except Exception, e:
logger.error(e)
return False
def scanSiteMain(self, siteId):
try:
logger.debug("start to scan site, siteId: %s" % (siteId))
if siteId == None:
return False
dao = MysqlDao()
siteObj = dao.getSiteData(siteId)
if siteObj == None:
logger.error("start to get site config exception, siteId: %s" %
(siteId))
return False
#scheme
scheme = siteObj['scheme'].encode('utf8')
#ip address
ip = siteObj['ip'].encode('utf8')
#site domain
domain = siteObj['domain'].encode('utf8')
#site scan state
state = siteObj['state']
#site path
path = siteObj['path'].encode('utf8')
#site title
title = siteObj['title'].encode('utf8')
#site type
siteType = siteObj['site_type'].encode('utf8')
#site cookie
cookie = siteObj['cookie'].encode('utf8')
#site include url
includeUrl = siteObj['include_url'].encode('utf8')
if includeUrl == '':
includeUrl = []
else:
includeUrl = json.read(includeUrl)
#site exclude url
excludeUrl = siteObj['exclude_url'].encode('utf8')
if excludeUrl == '':
excludeUrl = []
else:
excludeUrl = json.read(excludeUrl)
#scan progress
progress = siteObj['progress'].encode('utf8')
#site scan policy
policy = siteObj['policy']
if state == 1:
self.finishSiteScan(siteId, ip)
return True
#在DNS配置文件中加入这个域名的DNS信息
# self.threadLock.acquire()
# self.updateHosts(ip, domain, self.taskId, siteId, 'add')
# self.threadLock.release()
'''
# 注释此段，在后文（代码第700行附近）重写网站存活性检测，提高稳健性，并将结果写入报告 20170804
flag = res = content = checkOk = None
target = []
target.append("%s://%s%s"%(scheme,domain,path))
# -------------UPDATE BY MCJ 扫到site即可开始扫描，无需再检测网站状态
checkOk = True
# for url in target:
# flag, res, content = self.PreSiteScan(url)
# if not flag:
# continue
# else:
# if self.checkSiteWorkMode(res, title) == False:
# continue
# else:
# checkOk = 1
# break
# ----------
if not checkOk:
self.updateSiteException("网站无法访问", siteId, ip)
return
else:
siteCode = self.getSiteCode(content)
if title == "" and res and res.has_key('status') and res['status'] == '200':
title = self.updateSiteTitle(content, siteId)
if siteType == "":
siteType = self.updateSiteType(res, siteId)
if siteCode == "":
siteCode = self.getSiteCode(content)
'''
if self.taskCnf['web_scan_timeout']:
socket.setdefaulttimeout(self.taskCnf['web_scan_timeout'])
siteDb = {'state': 0, 'exception': ''}
if siteObj['start_time'] is None or siteObj[
'start_time'] == '0000-00-00 00:00:00':
siteDb['start_time'] = time.strftime("%Y-%m-%d %X",
time.localtime())
if siteObj['progress'] == '':
siteDb['progress'] = '0'
self.dao.updateData('sites', siteDb, {'id': siteId})
###############################
#policy:
# 1:快速扫描，只扫描指定的域名
# 2:完全扫描，扫描指定的域名，并且扫描二级域名
# 3:扫描指定目录及子目录
# 4:扫描指定的URL，这个情况下，不需要爬虫
# 5:通过域名反查得到的域名
# 6:登陆型扫描
###############################
## 禁用spider by mcj
# if self.taskCnf['spider_enable'] == 1 and siteObj['spider_state'] == 0:
# logger.debug('spider is start')
#
# progress = '0'
#
# self.dao.deleteData('web_result', {'site_id':siteId})
# self.dao.deleteData('web_result_data', {'site_id':siteId})
# self.dao.deleteData('spider_url', {'site_id':siteId})
#
# #开启爬虫，当扫描指定的URL时，不需要爬虫
# if siteObj['policy'] != 4:
# spiderCnf = {}
# spiderCnf['taskId'] = self.taskId
# spiderCnf['assetTaskId'] = self.assetTaskId
# spiderCnf['siteId'] = siteId
# spiderCnf['spiderUrlCount'] = self.taskCnf['spider_url_count']
# spiderCnf['webScanTime'] = self.taskCnf['web_scan_timeout']
# spiderCnf['policy'] = siteObj['policy']
# spiderCnf['scheme'] = siteObj['scheme'].encode('utf8')
# spiderCnf['domain'] = domain
# spiderCnf['path'] = path
# spiderCnf['maxTimeCount'] = 30
# spiderCnf['webScanTimeout'] = self.taskCnf['web_scan_timeout']
# spiderCnf['endTime'] = time.time() + 1800
# spiderCnf['maxnum'] = self.taskCnf['spider_url_count']
# spiderCnf['title'] = title
# spiderCnf['ip'] = ip
# spiderCnf['cookie'] = cookie
# spiderCnf['webSearchSiteState'] = self.taskCnf['web_search_site_state']
# spiderCnf['webSearchSiteTimeout'] = self.taskCnf['web_search_site_timeout']
# spiderCnf['includeUrl'] = includeUrl
# spiderCnf['excludeUrl'] = excludeUrl
# spiderCnf['downloadDir'] = SCANER_SPIDER_DOWNLOAD_DIR
#
# if self.taskCnf['spider_type'] == 2:
# import Spider2 as Spider
# else:
# import Spider
#
# logger.debug("spiderCnf start")
# logger.debug(spiderCnf)
# logger.debug("spiderCnf end")
# spider = Spider.Spider(spiderCnf)
# spider.start()
#
# logger.debug('spider is end')
self.dao.updateData('sites', {'spider_state': 1}, {'id': siteId})
siteCnf = dao.getSiteData(siteId)
domain = siteCnf['domain'].encode('utf8')
path = siteCnf['path'].encode('utf8')
#检测网站的状态，有的网站访问后直接访问500或者其他的情况。
if self.checkSiteWorkMode({}, title) == False:
self.finishSiteScan(siteId, ip)
return
logger.debug('get site scan config')
scanCnf = {}
scanCnf['taskId'] = self.taskId
scanCnf['assetTaskId'] = self.assetTaskId
scanCnf['siteId'] = siteId
scanCnf['maxThread'] = 10
scanCnf['scriptThread'] = 10
scanCnf['webTimeout'] = self.taskCnf['web_scan_timeout']
scanCnf['ip'] = ip
# 新增源站ip参数，add by mcj
target = json.read(str(self.taskCnf['target']))
source_ip = target[0].get('source_ip')
if source_ip:
scanCnf['source_ip'] = source_ip
scanCnf['scheme'] = scheme
scanCnf['domain'] = domain
scanCnf['path'] = path
scanCnf['errorCount'] = 0
scanCnf['errorLenDict'] = {}
scanCnf['maxTimeoutCount'] = 20
scanCnf['cookie'] = cookie
scanCnf['len404'] = []
scanCnf['isForce'] = 0
scanCnf['excludeUrl'] = excludeUrl
scanCnf['threadLock'] = threading.Lock()
scanCnf['isstart'] = '1'
# ----------- 判断网站存活性, 如存活，获取cookie等内容 by lichao
if source_ip:
test_url = "%s://%s%s" % (scheme, source_ip, path)
else:
test_url = "%s://%s%s" % (scheme, domain, path)
test_header = {'Host': domain}
checkOk, siteCode = False, None
for i in range(3):
try:
http = HttpRequest({
'domain': domain,
'timeout': 15,
'follow_redirects': True,
'cookie': cookie
})
res, content = http.request(test_url,
'GET',
headers=test_header)
if self.checkSiteWorkMode(res, title):
siteCode = self.getSiteCode(content)
if not title:
title = self.updateSiteTitle(content, siteId)
if not siteType:
siteType = self.updateSiteType(res, siteId)
if not cookie:
cookie = res.get('set-cookie')
checkOk = True
break
else:
sleep(5)
except:
sleep(5)
if cookie:
scanCnf['cookie'] = cookie
if title:
scanCnf['title'] = title
if siteType:
scanCnf['siteType'] = siteType
if siteCode:
scanCnf['siteCode'] = siteCode
if not checkOk:
self.updateSiteException("网站无法访问", siteId, ip)
# --------------------------------------------
# ------------------- 检测网站建站系统指纹 by lichao 预留功能，暂时没有用到
# if checkOk:
# from engine.engine_utils.check_web_fingerprint import web_frame_fingerprint
# scanCnf['web_frame'] = web_frame_fingerprint(ob=scanCnf)
# -------------------
# ----------- get sites_dirs by mcj
site_dirs = get_site_dirs(self.taskId)
scanCnf['site_dirs'] = site_dirs
# -----------
# ---------------- get web fingerprint by lichao
if checkOk:
scanCnf['webServer'] = web_server_fingerprint(
scheme, source_ip, domain,
path) # 'apache|nginx|iis|unknown'
scanCnf['os'] = os_fingerprint(
source_ip) # 'linux|windows|unknown'
# -----------------
# ---------------- verify 404 page and waf page by lichao
scanCnf['404_page'], scanCnf['app_404_page'], scanCnf[
'waf_page'] = get_invaild_page(scheme, source_ip, domain,
siteType)
scanCnf['404_page']['similar_rate'] = 0.8
scanCnf['app_404_page']['similar_rate'] = 0.8
scanCnf['waf_page']['similar_rate'] = 0.8
# ---------------------------
# 判断该域名扫描进度，加载未扫描的漏洞ID
logger.debug('load unscaned script start')
scanVulList = []
progress = progress.split('|')
for vulId in self.taskCnf['vulList']:
if vulId not in progress:
scanVulList.append(vulId)
logger.debug('script scan is start')
if len(scanVulList) > 0:
urlList = []
if policy == 4:
for url in includeUrl:
if url in excludeUrl:
continue
t = url.split('?')
url = t[0]
params = ''
if len(t) > 1:
params = t[1]
urlList.append({
'url': url,
'params': params,
'method': 'get'
})
else:
res = self.dao.getUrlList(siteId)
for r in res:
url = r['url'].encode('utf8')
if nonascii(url): url = safeUrlString(url)
urlList.append({
'url': url,
'params': r['params'].encode('utf8'),
'method': r['method'].encode('utf8'),
'refer': r['refer'].encode('utf8')
})
# ----------- 检测网站存活性 by lichao 拿到检测网站存活性的插件id
check_ok_vul_id = ""
db_session = DBSession()
try:
vul = db_session.query(WebVulList).filter(
WebVulList.script == 'check_web_alive').first()
check_ok_vul_id = str(vul.id)
except Exception, e:
logger.error(e)
db_session.close()
# -----------
for vulId in scanVulList:
from time import time as during_time
t1 = during_time()
vulId = vulId.replace(" ", "")
if vulId == "":
continue
# ----------- 检测网站存活性 by lichao
if not checkOk and len(urlList) <= 1: # 判断网站无法访问
if vulId != check_ok_vul_id: # 网站无法访问时，只运行 check_web_alive 这一个插件
continue
else: # 网站可以访问时
if vulId == check_ok_vul_id: # 网站可以访问时，不运行 check_web_alive 这个插件
continue
# ------------
progress.append(vulId)
self.dao.updateData('sites',
{'progress': '|'.join(progress)},
{'id': siteId})
self.dao.deleteData('web_result', {
'vul_id': vulId,
'site_id': siteId
})
scanCnf['vulId'] = vulId
scanCnf['vulName'] = self.taskCnf['vulDict'][vulId][
'vul_name']
scanCnf['level'] = self.taskCnf['vulDict'][vulId][
'level'].encode('utf8')
scanCnf['scanType'] = self.taskCnf['vulDict'][vulId][
'scan_type']
scanCnf['script'] = self.taskCnf['vulDict'][vulId][
'script']
scanCnf['status'] = '0'
scanCnf['endTime'] = time.time() + 1800
scanCnf['timeoutCount'] = 0
#测试爬虫爬出来的路径
if scanCnf['scanType'] == 1:
scanCnf['queue'] = Queue()
for r in urlList:
scanCnf['queue'].put(r)
scanUrlScript = ScanScriptForUrl(scanCnf)
scanUrlScript.start()
#如果只测试指定的URL则不需要运行测试域名和测试漏洞库
if policy != 4:
#测试域名
if scanCnf['scanType'] == 2:
scanDomainScript = ScanScriptForDomain(scanCnf)
scanDomainScript.start()
duration = during_time() - t1
# -----------统计插件运行时间 by mcj
try:
from common.plugin_speed import PluginSpeed
db_session = DBSession()
plu_speed = PluginSpeed(self.taskId, vulId, duration)
db_session.add(plu_speed)
db_session.commit()
db_session.close()
except Exception, e:
logger.info(str(e))
db_session.rollback()
db_session.close()
# -----------统计插件运行时间 by mcj
if not checkOk and len(urlList) <= 1:
break
urlList = []
def init(self):
try:
if self.action == 'reinit':
logger.debug('start to reinit task')
reinitDb = {'state':1,'init_state':0,'prescan_state':0,'web_scan_state':0,'weak_pwd_scan_state':0,'port_scan_state':0,'host_scan_state':0,'start_time':'0000-00-00 00:00:00','end_time':'0000-00-00 00:00:00'}
self.dao.updateData('task', reinitDb, {'id':self.taskId})
#end if
currentTime = time.strftime("%Y-%m-%d %X",time.localtime())
self.dao.updateData('task', {'state':2,'start_time':currentTime,'end_time':'0000-00-00 00:00:00'}, {'id':self.taskId,'state':1})
self.dao.updateData('task', {'state':3,'end_time':currentTime}, {'id':self.taskId,'init_state':1,'prescan_state':1,'web_scan_state':1,'weak_pwd_scan_state':1,'port_scan_state':1,'host_scan_state':1})
taskCnf = self.dao.getTaskData(self.taskId)
self.assetTaskId = taskCnf['asset_task_id']
if taskCnf['init_state'] == 1:
return
#end if
target = taskCnf['target'].encode('utf8')
if target == '':
target = []
else:
target = json.read(target.encode('utf8'))
#end if
#clear host_infos
self.dao.deleteData('host_infos', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear host_ports
self.dao.deleteData('host_ports', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear sites
self.dao.deleteData('sites', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear spider_url
self.dao.deleteData('spider_url', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear web_result
self.dao.deleteData('web_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear web_result_data
self.dao.deleteData('web_result_data', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear host_result
self.dao.deleteData('host_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
#clear weak_pwd_result
self.dao.deleteData('weak_pwd_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId})
siteList = []
ipList = []
for item in target:
try:
domain = ''
ip = ''
if item.has_key('domain'):
domain = item['domain']
#end if
if item.has_key('ip'):
ip = item['ip']
#end if
if domain == '' and ip == '':
continue
#end if
if ip == '':
ip = domainToip(domain)
if ip == False:
continue
#end if
#end if
ipDb = {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip}
if ip not in ipList:
hostId = self.dao.insertData('host_infos', ipDb)
if hostId > 0:
ipList.append(ip)
#end if
#end if
except Exception, e1:
logger.error(e1)
#end try
#end for
self.dao.updateData('task', {'init_state':1}, {'id':self.taskId})