def init(self): try: if self.action == 'restart': logger.debug('restart to scan weak pwd') self.dao.deleteData('weak_pwd_result', { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId }) self.dao.updateData('host_infos', {'weak_pwd_scan_state': 0}, { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId }) self.dao.updateData('task', {'weak_pwd_scan_state': 0}, {'id': self.taskId}) #end if self.taskCnf = self.dao.getTaskData(self.taskId) if self.taskCnf == False: return False #end if self.assetTaskId = self.taskCnf['asset_task_id'] self.thread = self.taskCnf['weak_pwd_scan_thread'] self.timeout = self.taskCnf['weak_pwd_scan_timeout'] self.policy = self.taskCnf['weak_pwd_scan_policy'].split(',') return True except Exception, e: logger.error(e) return False
def run(self): try: logger.debug('Host scan is start') if self.init() == False: return #end if logger.debug('Host scan init is success') if self.taskCnf['host_scan_enable'] == 0: self.finish() return #end if logger.debug('Host scan is enable') if self.taskCnf['host_scan_enable'] == 1 and self.taskCnf[ 'host_scan_state'] == 1: self.finish() return #end if logger.debug('Host scan is not finished') res = self.dao.getData( 'host_infos', { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId, 'host_scan_state': 0 }) if not res or len(res) <= 0: self.finish() return #end if logger.debug('find host_scan target to scan') for row in res: self.target.put(row['ip']) #end for threadList = [] for i in range(self.thread): t = HostScanThread(self.taskCnf, self.target) threadList.append(t) #end for for t in threadList: t.start() #end for for t in threadList: t.join() #end for self.finish() except Exception, e: logger.error(e)
def run(self): try: cmd = "%s -sS -p 1-9999 -V -i %s %s" % (NASL_EXE, self.vulIds, self.ip) logger.debug(cmd) popenData = popen(cmd) res = self.handleResult(popenData) self.updateHostResult(res) except Exception, e: logger.error(e)
def init(self): try: if self.action == 'restart': logger.debug('restart to scan port') self.dao.deleteData('host_ports', { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId }) self.dao.updateData('host_infos', {'port_scan_state': 0}, { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId }) self.dao.updateData('task', {'port_scan_state': 0}, {'id': self.taskId}) #end if self.taskCnf = self.dao.getTaskData(self.taskId) if self.taskCnf == False: return False #end if logger.debug('get task config success') self.assetTaskId = self.taskCnf['asset_task_id'] self.thread = self.taskCnf['port_scan_thread'] self.timeout = self.taskCnf['port_scan_timeout'] self.policy = self.taskCnf['port_scan_policy'] res = self.dao.getRowData('port_scan_policy', {'id': self.policy}) if res and len(res) > 0: self.ports = res['ports'] logger.debug('get port scan policy success') #end if if self.taskCnf['port_scan_enable'] == 1 and self.taskCnf[ 'port_scan_state'] == 0: logger.debug('port scan init success') return True #end if logger.debug('port scan is finished') self.dao.updateData('host_infos', {'port_scan_state': 1}, { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId, 'state': 1 }) return False except Exception, e: logger.error(e) return False
def __init__(self, taskId, assetTaskId, taskCnf, threadLock): logger.debug("__init__ scansite") try: threading.Thread.__init__(self) self.module = self.__class__.__name__ self.taskId = taskId self.assetTaskId = assetTaskId self.taskCnf = taskCnf self.threadLock = threadLock self.threadName = threading.currentThread().getName() self.dao = MysqlDao() self.count = 0 except Exception, e: logger.exception(e)
def main(self): try: # add params domain use for init host_scan of the header by mcj http = HttpRequest({ 'domain': self.domain, 'timeout': self.webTimeout, 'follow_redirects': True, 'cookie': self.cookie }) logger.debug("from plugins.%s import *" % (self.script)) exec("from plugins.%s import *" % (self.script)) #临时计数器 temp_num = 0 #开始扫描 while True: try: if self.urlQueue.qsize() < 1: break list = [] try: item = self.urlQueue.get_nowait() except Exception, e2: break url = item['url'] if item['method'] == 'get' and item['params'] != '': url = "%s?%s" % (url, item['params']) if self.checkExcludeUrl(self.excludeUrl, url): continue list = run_url(http, self.scanCnf, item) if list and len(list) > 0: temp_num += len(list) #对于邮件信息泄露和HTML注释信息泄露做了部分特殊处理 if self.script in [ 'HtmlSourceLeakScript', 'EmailDiscloseScript' ] and temp_num > 10: break self.threadLock.acquire() self.updateResult(list) self.threadLock.release() except Exception, e1: logger.error(e1)
def start(self): try: if self.script == "": return # add params domain use for init host_scan of the header by mcj http = HttpRequest({ 'domain': self.domain, 'timeout': self.webTimeout, 'follow_redirects': True }) logger.debug("from plugins.%s import *" % (self.script)) exec("from plugins.%s import *" % (self.script)) #开始扫描 list = run_domain(http, self.scanCnf) if list and len(list) > 0: self.updateResult(list) except Exception, e: logger.error(e)
def init(self): try: self.taskCnf = self.dao.getTaskData(self.taskId) if self.taskCnf == False: return False self.assetTaskId = self.taskCnf['asset_task_id'] self.taskCnf['vulDict'] = {} self.taskCnf['vulList'] = [] vulList = self.dao.getWebVulByPolicy( self.taskCnf['web_scan_policy']) for row in vulList: vulId = str(row['vul_id']) self.taskCnf['vulDict'][vulId] = row self.taskCnf['vulList'].append(vulId) return True except Exception, e: logger.debug(e) return False
def run(self): try: logger.debug('start to scan site') i = 0 while True: i += 1 logger.debug("try to find new site, i: %d" % (i)) try: self.threadLock.acquire() checkSiteUnscand = self.checkSiteUnScaned() self.threadLock.release() if checkSiteUnscand: break siteId = None try: siteId = siteQueue.get(True, 30) except Exception, ge: pass #开始这个域名的扫描 if siteId and self.checkSiteId(siteId): self.count = 0 logger.debug("find new site to scan, siteId: %s" % (siteId)) self.scanSiteMain(str(siteId)) self.checkExceptionSite() except Exception, e1: logger.error(e1) self.count += 1
def scanMain(self): try: logger.debug('weak pwd scan main is start') while True: if self.target.qsize() < 1: break #end if ip = self.target.get_nowait() if ip and ip != '': logger.debug("weak pwd scan man is scaning ip: %s" % (ip)) scanList = self.checkIpPort(ip) if scanList and len(scanList) > 0: for line in scanList: self.crack(ip, line) #end for #end if #更新主机状态 self.updateHostState(ip) #end if #end while except Exception, e: logger.error(e)
def run(self): try: while True: if self.queue.qsize() < 1: print "prescan thread exit" break #end if ip = self.queue.get_nowait() if not ip or ip == "": print "prescan thread exit" break #end if if self.checkIp(ip): self.dao.updateData('host_infos', {'state':1}, {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip}) else: logger.debug("ip is not alive: %s" % (ip)) self.dao.updateData('host_infos', {'state':2,'port_scan_state':1,'host_scan_state':1,'web_scan_state':1,'weak_pwd_scan_state':1}, {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip}) #end if #end while except Exception,e: logger.error(e)
def run(self): try: logger.debug('start to search site') # 获取上一次未扫描完成的域名 logger.debug('taskId %s ' % self.taskId) siteIds = self.dao.getUnscandSite(self.taskId, self.assetTaskId) logger.debug(siteIds) for siteId in siteIds: logger.debug('siteQueue put %s ' % siteId) siteQueue.put(siteId) if self.taskCnf['web_search_site_state'] == 0: target = self.taskCnf['target'].encode('utf8') if target == '': target = [] else: target = json.read(target) ipList = [] logger.debug('target: ' + jsonSys.dumps(target)) for row in target: try: siteObj = { 'scheme': 'http', 'domain': '', 'path': '/', 'ip': '', 'title': '', 'policy': '1', 'include_url': '', 'exclude_url': '', 'cookie': '', 'sub_domain_scan': 0, 'ip_domain_scan': 0 } if row.has_key('scheme'): siteObj['scheme'] = row['scheme'] if row.has_key('domain') == False: logger.error("can not get domain") continue siteObj['domain'] = row['domain'] if row.has_key('path'): siteObj['path'] = row['path'] if row.has_key('ip'): # 支持配置自定义IP指向 ip = row['ip'] else: ip = domainToip(siteObj['domain']) if ip == False: logger.error("can not get ip, domain: %s" % (siteObj['domain'])) continue siteObj['ip'] = ip if row.has_key('title'): siteObj['title'] = row['title'] if row.has_key('policy'): siteObj['policy'] = row['policy'] else: siteObj['policy'] = 1 if row.has_key('include_url'): siteObj['include_url'] = json.write( row['include_url']) if row.has_key('exclude_url'): siteObj['exclude_url'] = json.write( row['exclude_url']) if row.has_key('cookie'): siteObj['cookie'] = row['cookie'] # 是否扫描二级域名 if row.has_key('sub_domain_scan'): siteObj['sub_domain_scan'] = row['sub_domain_scan'] # 该IP的其他域名扫描 if row.has_key('ip_domain_scan'): siteObj['ip_domain_scan'] = row['ip_domain_scan'] if siteObj['ip_domain_scan'] == 1: # 根据IP获取IP下的域名 if ip not in ipList: otherSites = self.searchOtherSiteInIp(ip) target.extend(otherSites) ipList.append(ip) self.updateTaskSites(siteObj) except Exception, ee: logger.error(ee) self.finish() logger.debug('end to search site')
def crack(self, ip, type): try: logger.debug('start to crack ip: %s, vulName: %s' % (ip, type)) dicFile = "dict/%s.dic" % (type) f = open(dicFile, "r") lines = f.readlines() startTime = time.time() if type == "telnet": telnet_check = Telnet_login(ip) #end if for line in lines: try: if int(time.time() - startTime) > self.timeout: break #end if if type == "vnc": username = "" password = line.strip() password = password.replace("\\", "\\\\") password = password.replace("`", "\\`") password = password.replace("\"", "\\\"") else: up = line.split(":", 1) if len(up) < 2: continue #end if username = up[0].strip() password = up[1].strip() username = username.replace("\\", "\\\\") username = username.replace("`", "\\`") username = username.replace("\"", "\\\"") password = password.replace("\\", "\\\\") password = password.replace("`", "\\`") password = password.replace("\"", "\\\"") #end if cmd = "" if type == "vnc": if password == "{NULL}": cmd = "hydra -t 1 -sid=%s -p \"\" %s %s" % ( self.taskId, ip, type) else: cmd = "hydra -t 1 -sid=%s -p \"%s\" %s %s" % ( self.taskId, password, ip, type) #end if elif type == "oracle": if password == "{NULL}": cmd = "hydra -t 1 -sid=%s -l \"%s\" -e n %s %s ORCL" % ( self.taskId, username, ip, type) else: cmd = "hydra -t 1 -sid=%s -l \"%s\" -p \"%s\" %s %s ORCL" % ( self.taskId, username, password, ip, type) #end if else: if password == "{NULL}": cmd = "hydra -t 1 -sid=%s -l \"%s\" -e n %s %s" % ( self.taskId, username, ip, type) else: cmd = "hydra -t 1 -sid=%s -l \"%s\" -p \"%s\" %s %s" % ( self.taskId, username, password, ip, type) #end if #end if logger.debug(cmd) (output, exitstatus) = run(cmd, withexitstatus=1, timeout=20) logger.debug(output) #m = self.compiledRule.findall(output) #print output #print m #if len(m) == 1: if output.find('1 valid password found') > 0: #v = m[0] #username = v[3].strip() #password = v[4].strip() if type == "telnet": telnet_passwd = '' if len( password) == 0 else password if not telnet_check.check(username, telnet_passwd): continue #end if #end if if len(password) == 0: self.updateResult(ip, type.upper(), username, "空密码") if type.upper() == "FTP": self.updateResult(ip, type.upper(), "ftp", "ftp") #end if else: self.updateResult(ip, type.upper(), username, password) #end if return #end if except Exception, e1: logger.error(e) continue #end try #end for except Exception, e: logger.error(e)
def scanSiteMain(self, siteId): try: logger.debug("start to scan site, siteId: %s" % (siteId)) if siteId == None: return False dao = MysqlDao() siteObj = dao.getSiteData(siteId) if siteObj == None: logger.error("start to get site config exception, siteId: %s" % (siteId)) return False #scheme scheme = siteObj['scheme'].encode('utf8') #ip address ip = siteObj['ip'].encode('utf8') #site domain domain = siteObj['domain'].encode('utf8') #site scan state state = siteObj['state'] #site path path = siteObj['path'].encode('utf8') #site title title = siteObj['title'].encode('utf8') #site type siteType = siteObj['site_type'].encode('utf8') #site cookie cookie = siteObj['cookie'].encode('utf8') #site include url includeUrl = siteObj['include_url'].encode('utf8') if includeUrl == '': includeUrl = [] else: includeUrl = json.read(includeUrl) #site exclude url excludeUrl = siteObj['exclude_url'].encode('utf8') if excludeUrl == '': excludeUrl = [] else: excludeUrl = json.read(excludeUrl) #scan progress progress = siteObj['progress'].encode('utf8') #site scan policy policy = siteObj['policy'] logger.debug("scanSiteMain siteId: %s" % (siteId)) if state == 1: self.finishSiteScan(siteId, ip) return True #在DNS配置文件中加入这个域名的DNS信息 self.threadLock.acquire() self.updateHosts(ip, domain, self.taskId, siteId, 'add') self.threadLock.release() flag = res = content = checkOk = None target = [] logger.debug("scanSiteMain siteId: %s preSiteScan before" % (siteId)) target.append("%s://%s%s" % (scheme, domain, path)) for url in target: flag, res, content = self.PreSiteScan(url) if not flag: continue else: if self.checkSiteWorkMode(res, title) == False: continue else: checkOk = 1 break if not checkOk: self.updateSiteException("网站无法访问", siteId, ip) return else: siteCode = self.getSiteCode(content) if title == "" and res and res.has_key( 'status') and res['status'] == '200': title = self.updateSiteTitle(content, siteId) if siteType == "": siteType = self.updateSiteType(res, siteId) if siteCode == "": siteCode = self.getSiteCode(content) if self.taskCnf['web_scan_timeout']: socket.setdefaulttimeout(self.taskCnf['web_scan_timeout']) siteDb = {'state': 0, 'exception': ''} if siteObj['start_time'] is None or siteObj[ 'start_time'] == '0000-00-00 00:00:00': siteDb['start_time'] = time.strftime("%Y-%m-%d %X", time.localtime()) if siteObj['progress'] == '': siteDb['progress'] = '0' self.dao.updateData('sites', siteDb, {'id': siteId}) logger.debug("scanSiteMain siteId: %s policy before" % (siteId)) ############################### #policy: # 1:快速扫描,只扫描指定的域名 # 2:完全扫描,扫描指定的域名,并且扫描二级域名 # 3:扫描指定目录及子目录 # 4:扫描指定的URL,这个情况下,不需要爬虫 # 5:通过域名反查得到的域名 # 6:登陆型扫描 ############################### if self.taskCnf['spider_enable'] == 1 and siteObj[ 'spider_state'] == 0: logger.debug('spider is start') progress = '0' logger.debug("scanSiteMain siteId: %s cleandata before" % (siteId)) self.dao.deleteData('web_result', {'site_id': siteId}) self.dao.deleteData('web_result_data', {'site_id': siteId}) self.dao.deleteData('spider_url', {'site_id': siteId}) #开启爬虫,当扫描指定的URL时,不需要爬虫 if siteObj['policy'] != 4: spiderCnf = {} spiderCnf['taskId'] = self.taskId spiderCnf['assetTaskId'] = self.assetTaskId spiderCnf['siteId'] = siteId spiderCnf['spiderUrlCount'] = self.taskCnf[ 'spider_url_count'] spiderCnf['webScanTime'] = self.taskCnf['web_scan_timeout'] spiderCnf['policy'] = siteObj['policy'] spiderCnf['scheme'] = siteObj['scheme'].encode('utf8') spiderCnf['domain'] = domain spiderCnf['path'] = path spiderCnf['maxTimeCount'] = 30 spiderCnf['webScanTimeout'] = self.taskCnf[ 'web_scan_timeout'] spiderCnf['endTime'] = time.time() + 1800 spiderCnf['maxnum'] = self.taskCnf['spider_url_count'] spiderCnf['title'] = title spiderCnf['ip'] = ip spiderCnf['cookie'] = cookie spiderCnf['webSearchSiteState'] = self.taskCnf[ 'web_search_site_state'] spiderCnf['webSearchSiteTimeout'] = self.taskCnf[ 'web_search_site_timeout'] spiderCnf['includeUrl'] = includeUrl spiderCnf['excludeUrl'] = excludeUrl spiderCnf['downloadDir'] = './log/' logger.debug("scanSiteMain siteId: %s startSpider before" % (siteId)) if self.taskCnf['spider_type'] == 2: import Spider2 as Spider else: import Spider logger.debug("spiderCnf start") logger.debug(spiderCnf) logger.debug("spiderCnf end") spider = Spider.Spider(spiderCnf) spider.start() logger.debug('spider is end') self.dao.updateData('sites', {'spider_state': 1}, {'id': siteId}) ####################################### 至此蜘蛛结束,后面是扫描的工作 ############################################# return True siteCnf = dao.getSiteData(siteId) domain = siteCnf['domain'].encode('utf8') path = siteCnf['path'].encode('utf8') title = siteCnf['title'] if title != '': title = title.encode('utf8') #检测网站的状态,有的网站访问后直接访问500或者其他的情况。 if self.checkSiteWorkMode({}, title) == False: self.finishSiteScan(siteId, ip) return logger.debug('get site scan config') scanCnf = {} scanCnf['taskId'] = self.taskId scanCnf['assetTaskId'] = self.assetTaskId scanCnf['siteId'] = siteId scanCnf['maxThread'] = 10 scanCnf['scriptThread'] = 10 scanCnf['webTimeout'] = self.taskCnf['web_scan_timeout'] scanCnf['ip'] = ip scanCnf['scheme'] = scheme scanCnf['domain'] = domain scanCnf['path'] = path scanCnf['cookie'] = cookie scanCnf['errorCount'] = 0 scanCnf['errorLenDict'] = {} scanCnf['siteType'] = siteType scanCnf['maxTimeoutCount'] = 20 scanCnf['siteCode'] = siteCode scanCnf['cookie'] = cookie scanCnf['len404'] = [] scanCnf['isForce'] = 0 scanCnf['excludeUrl'] = excludeUrl scanCnf['threadLock'] = threading.Lock() scanCnf['isstart'] = '1' #判断该域名扫描进度,加载未扫描的漏洞ID logger.debug('load unscaned script start') scanVulList = [] progress = progress.split('|') for vulId in self.taskCnf['vulList']: if vulId not in progress: scanVulList.append(vulId) logger.debug('script scan is start') if len(scanVulList) > 0: urlList = [] if policy == 4: for url in includeUrl: if url in excludeUrl: continue t = r.split('?') url = t[0] params = '' if len(t) > 1: params = t[1] urlList.append({ 'url': url, 'params': params, 'method': 'get' }) else: res = self.dao.getUrlList(siteId) for r in res: url = r['url'].encode('utf8') if nonascii(url): url = safeUrlString(url) urlList.append({ 'url': url, 'params': r['params'].encode('utf8'), 'method': r['method'].encode('utf8'), 'refer': r['refer'].encode('utf8') }) for vulId in scanVulList: vulId = vulId.replace(" ", "") if vulId == "": continue progress.append(vulId) self.dao.updateData('sites', {'progress': '|'.join(progress)}, {'id': siteId}) self.dao.deleteData('web_result', { 'vul_id': vulId, 'site_id': siteId }) scanCnf['vulId'] = vulId scanCnf['vulName'] = self.taskCnf['vulDict'][vulId][ 'vul_name'] scanCnf['level'] = self.taskCnf['vulDict'][vulId][ 'level'].encode('utf8') scanCnf['scanType'] = self.taskCnf['vulDict'][vulId][ 'scan_type'] scanCnf['script'] = self.taskCnf['vulDict'][vulId][ 'script'] scanCnf['status'] = '0' scanCnf['endTime'] = time.time() + 1800 scanCnf['timeoutCount'] = 0 #测试爬虫爬出来的路径 if scanCnf['scanType'] == 1: scanCnf['queue'] = Queue() for r in urlList: scanCnf['queue'].put(r) scanUrlScript = ScanScriptForUrl(scanCnf) scanUrlScript.start() #如果只测试指定的URL则不需要运行测试域名和测试漏洞库 if policy != 4: #测试域名 if scanCnf['scanType'] == 2: scanDomainScript = ScanScriptForDomain(scanCnf) scanDomainScript.start() urlList = [] #结束扫描 self.finishSiteScan(siteId, ip) self.threadLock.acquire() self.updateHosts(ip, domain, self.taskId, siteId, 'remove') self.threadLock.release() return None except Exception, e: logger.exception(e) return siteId
def init(self): logger.debug("init scansite") try: pass except Exception, e: logger.exception(e)
def updateTaskSites(self, siteObj): try: logger.debug('start to add new site') try: scheme, domain, path = getRedirect( "%s://%s%s" % (siteObj['scheme'], siteObj['domain'], siteObj['path'])) except Exception, e1: scheme = siteObj['scheme'] domain = siteObj['domain'] path = siteObj['path'] pass if scheme != 'http' and scheme != 'https': siteObj['scheme'] = 'http' else: siteObj['scheme'] = scheme if not path or path == '': siteObj['path'] = path siteExistObj = { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId, 'scheme': siteObj['scheme'], 'domain': siteObj['domain'], 'path': siteObj['path'] } if self.dao.getDataCount('sites', siteExistObj) > 0: return True siteObj['task_id'] = self.taskId siteObj['asset_task_id'] = self.assetTaskId siteObj['progress'] = '' siteId = self.dao.insertData('sites', siteObj) if siteId == 0: logger.error('add new site to db exception') return False siteQueue.put(str(siteId)) ####### use the new spider add by mcj self.dao.updateData('spider_url', {'site_id': siteId}, {'task_id': self.taskId}) self.dao.updateData('spider_url_other', {'site_id': siteId}, {'task_id': self.taskId}) ############# ipExistObj = { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId, 'ip': siteObj['ip'] } if self.dao.getDataCount('host_infos', ipExistObj) < 1: currentTime = time.strftime("%Y-%m-%d %X", time.localtime()) hostObj = { 'task_id': self.taskId, 'asset_task_id': self.assetTaskId, 'ip': siteObj['ip'], 'start_time': currentTime } # mcj:insertData() takes exactly 3 arguments (2 given) add argument:host_infos self.dao.insertData('host_infos', hostObj) taskObj = { 'host_scan_state': 0, 'weak_pwd_scan_state': 0, 'port_scan_state': 0 } self.dao.updateData('task', taskObj, {'id': self.taskId})
logger.debug("try to find new site, i: %d" % (i)) try: self.threadLock.acquire() checkSiteUnscand = self.checkSiteUnScaned() self.threadLock.release() if checkSiteUnscand: break siteId = None try: siteId = siteQueue.get(True, 30) except Exception, ge: pass #开始这个域名的扫描 if siteId and self.checkSiteId(siteId): self.count = 0 logger.debug("find new site to scan, siteId: %s" % (siteId)) self.scanSiteMain(str(siteId)) self.checkExceptionSite() except Exception, e1: logger.error(e1) self.count += 1 if self.count > 2: break logger.debug('end to scan site') except Exception, e: logger.debug(e)
def run(self): try: logger.debug('clear last scan') if self.action == 'restart': if self.clean() == False: logger.error('清空上一次扫描失败') return False logger.debug('init scan config') #初始化扫描任务 if self.init() == False: logger.error('初始化扫描信息失败') return False logger.debug('check init state') if self.taskCnf['init_state'] == 0: logger.error('初始化失败') return False logger.debug('check if need to scan web vul') if self.taskCnf['web_scan_enable'] == 0: logger.debug('不需要扫描Web漏洞') self.finish() return True logger.debug('check web scan is finished') if self.taskCnf['web_scan_state'] == 1: logger.debug('Web扫描已完成') return True #调用获取域名的线程和扫描域名的线程 threadList = [] threadList.append(SearchSite(self.taskId, self.taskCnf)) for i in range(self.taskCnf['web_scan_thread']): threadList.append( ScanSite(self.taskId, self.assetTaskId, self.taskCnf, self.threadLock)) for t in threadList: t.start() for t in threadList: t.join() self.finish() except Exception, e: logger.error(e) return False
def scanSiteMain(self, siteId): try: logger.debug("start to scan site, siteId: %s" % (siteId)) if siteId == None: return False dao = MysqlDao() siteObj = dao.getSiteData(siteId) if siteObj == None: logger.error("start to get site config exception, siteId: %s" % (siteId)) return False #scheme scheme = siteObj['scheme'].encode('utf8') #ip address ip = siteObj['ip'].encode('utf8') #site domain domain = siteObj['domain'].encode('utf8') #site scan state state = siteObj['state'] #site path path = siteObj['path'].encode('utf8') #site title title = siteObj['title'].encode('utf8') #site type siteType = siteObj['site_type'].encode('utf8') #site cookie cookie = siteObj['cookie'].encode('utf8') #site include url includeUrl = siteObj['include_url'].encode('utf8') if includeUrl == '': includeUrl = [] else: includeUrl = json.read(includeUrl) #site exclude url excludeUrl = siteObj['exclude_url'].encode('utf8') if excludeUrl == '': excludeUrl = [] else: excludeUrl = json.read(excludeUrl) #scan progress progress = siteObj['progress'].encode('utf8') #site scan policy policy = siteObj['policy'] if state == 1: self.finishSiteScan(siteId, ip) return True #在DNS配置文件中加入这个域名的DNS信息 # self.threadLock.acquire() # self.updateHosts(ip, domain, self.taskId, siteId, 'add') # self.threadLock.release() ''' # 注释此段,在后文(代码第700行附近)重写网站存活性检测,提高稳健性,并将结果写入报告 20170804 flag = res = content = checkOk = None target = [] target.append("%s://%s%s"%(scheme,domain,path)) # -------------UPDATE BY MCJ 扫到site即可开始扫描,无需再检测网站状态 checkOk = True # for url in target: # flag, res, content = self.PreSiteScan(url) # if not flag: # continue # else: # if self.checkSiteWorkMode(res, title) == False: # continue # else: # checkOk = 1 # break # ---------- if not checkOk: self.updateSiteException("网站无法访问", siteId, ip) return else: siteCode = self.getSiteCode(content) if title == "" and res and res.has_key('status') and res['status'] == '200': title = self.updateSiteTitle(content, siteId) if siteType == "": siteType = self.updateSiteType(res, siteId) if siteCode == "": siteCode = self.getSiteCode(content) ''' if self.taskCnf['web_scan_timeout']: socket.setdefaulttimeout(self.taskCnf['web_scan_timeout']) siteDb = {'state': 0, 'exception': ''} if siteObj['start_time'] is None or siteObj[ 'start_time'] == '0000-00-00 00:00:00': siteDb['start_time'] = time.strftime("%Y-%m-%d %X", time.localtime()) if siteObj['progress'] == '': siteDb['progress'] = '0' self.dao.updateData('sites', siteDb, {'id': siteId}) ############################### #policy: # 1:快速扫描,只扫描指定的域名 # 2:完全扫描,扫描指定的域名,并且扫描二级域名 # 3:扫描指定目录及子目录 # 4:扫描指定的URL,这个情况下,不需要爬虫 # 5:通过域名反查得到的域名 # 6:登陆型扫描 ############################### ## 禁用spider by mcj # if self.taskCnf['spider_enable'] == 1 and siteObj['spider_state'] == 0: # logger.debug('spider is start') # # progress = '0' # # self.dao.deleteData('web_result', {'site_id':siteId}) # self.dao.deleteData('web_result_data', {'site_id':siteId}) # self.dao.deleteData('spider_url', {'site_id':siteId}) # # #开启爬虫,当扫描指定的URL时,不需要爬虫 # if siteObj['policy'] != 4: # spiderCnf = {} # spiderCnf['taskId'] = self.taskId # spiderCnf['assetTaskId'] = self.assetTaskId # spiderCnf['siteId'] = siteId # spiderCnf['spiderUrlCount'] = self.taskCnf['spider_url_count'] # spiderCnf['webScanTime'] = self.taskCnf['web_scan_timeout'] # spiderCnf['policy'] = siteObj['policy'] # spiderCnf['scheme'] = siteObj['scheme'].encode('utf8') # spiderCnf['domain'] = domain # spiderCnf['path'] = path # spiderCnf['maxTimeCount'] = 30 # spiderCnf['webScanTimeout'] = self.taskCnf['web_scan_timeout'] # spiderCnf['endTime'] = time.time() + 1800 # spiderCnf['maxnum'] = self.taskCnf['spider_url_count'] # spiderCnf['title'] = title # spiderCnf['ip'] = ip # spiderCnf['cookie'] = cookie # spiderCnf['webSearchSiteState'] = self.taskCnf['web_search_site_state'] # spiderCnf['webSearchSiteTimeout'] = self.taskCnf['web_search_site_timeout'] # spiderCnf['includeUrl'] = includeUrl # spiderCnf['excludeUrl'] = excludeUrl # spiderCnf['downloadDir'] = SCANER_SPIDER_DOWNLOAD_DIR # # if self.taskCnf['spider_type'] == 2: # import Spider2 as Spider # else: # import Spider # # logger.debug("spiderCnf start") # logger.debug(spiderCnf) # logger.debug("spiderCnf end") # spider = Spider.Spider(spiderCnf) # spider.start() # # logger.debug('spider is end') self.dao.updateData('sites', {'spider_state': 1}, {'id': siteId}) siteCnf = dao.getSiteData(siteId) domain = siteCnf['domain'].encode('utf8') path = siteCnf['path'].encode('utf8') #检测网站的状态,有的网站访问后直接访问500或者其他的情况。 if self.checkSiteWorkMode({}, title) == False: self.finishSiteScan(siteId, ip) return logger.debug('get site scan config') scanCnf = {} scanCnf['taskId'] = self.taskId scanCnf['assetTaskId'] = self.assetTaskId scanCnf['siteId'] = siteId scanCnf['maxThread'] = 10 scanCnf['scriptThread'] = 10 scanCnf['webTimeout'] = self.taskCnf['web_scan_timeout'] scanCnf['ip'] = ip # 新增源站ip参数,add by mcj target = json.read(str(self.taskCnf['target'])) source_ip = target[0].get('source_ip') if source_ip: scanCnf['source_ip'] = source_ip scanCnf['scheme'] = scheme scanCnf['domain'] = domain scanCnf['path'] = path scanCnf['errorCount'] = 0 scanCnf['errorLenDict'] = {} scanCnf['maxTimeoutCount'] = 20 scanCnf['cookie'] = cookie scanCnf['len404'] = [] scanCnf['isForce'] = 0 scanCnf['excludeUrl'] = excludeUrl scanCnf['threadLock'] = threading.Lock() scanCnf['isstart'] = '1' # ----------- 判断网站存活性, 如存活,获取cookie等内容 by lichao if source_ip: test_url = "%s://%s%s" % (scheme, source_ip, path) else: test_url = "%s://%s%s" % (scheme, domain, path) test_header = {'Host': domain} checkOk, siteCode = False, None for i in range(3): try: http = HttpRequest({ 'domain': domain, 'timeout': 15, 'follow_redirects': True, 'cookie': cookie }) res, content = http.request(test_url, 'GET', headers=test_header) if self.checkSiteWorkMode(res, title): siteCode = self.getSiteCode(content) if not title: title = self.updateSiteTitle(content, siteId) if not siteType: siteType = self.updateSiteType(res, siteId) if not cookie: cookie = res.get('set-cookie') checkOk = True break else: sleep(5) except: sleep(5) if cookie: scanCnf['cookie'] = cookie if title: scanCnf['title'] = title if siteType: scanCnf['siteType'] = siteType if siteCode: scanCnf['siteCode'] = siteCode if not checkOk: self.updateSiteException("网站无法访问", siteId, ip) # -------------------------------------------- # ------------------- 检测网站建站系统指纹 by lichao 预留功能,暂时没有用到 # if checkOk: # from engine.engine_utils.check_web_fingerprint import web_frame_fingerprint # scanCnf['web_frame'] = web_frame_fingerprint(ob=scanCnf) # ------------------- # ----------- get sites_dirs by mcj site_dirs = get_site_dirs(self.taskId) scanCnf['site_dirs'] = site_dirs # ----------- # ---------------- get web fingerprint by lichao if checkOk: scanCnf['webServer'] = web_server_fingerprint( scheme, source_ip, domain, path) # 'apache|nginx|iis|unknown' scanCnf['os'] = os_fingerprint( source_ip) # 'linux|windows|unknown' # ----------------- # ---------------- verify 404 page and waf page by lichao scanCnf['404_page'], scanCnf['app_404_page'], scanCnf[ 'waf_page'] = get_invaild_page(scheme, source_ip, domain, siteType) scanCnf['404_page']['similar_rate'] = 0.8 scanCnf['app_404_page']['similar_rate'] = 0.8 scanCnf['waf_page']['similar_rate'] = 0.8 # --------------------------- # 判断该域名扫描进度,加载未扫描的漏洞ID logger.debug('load unscaned script start') scanVulList = [] progress = progress.split('|') for vulId in self.taskCnf['vulList']: if vulId not in progress: scanVulList.append(vulId) logger.debug('script scan is start') if len(scanVulList) > 0: urlList = [] if policy == 4: for url in includeUrl: if url in excludeUrl: continue t = url.split('?') url = t[0] params = '' if len(t) > 1: params = t[1] urlList.append({ 'url': url, 'params': params, 'method': 'get' }) else: res = self.dao.getUrlList(siteId) for r in res: url = r['url'].encode('utf8') if nonascii(url): url = safeUrlString(url) urlList.append({ 'url': url, 'params': r['params'].encode('utf8'), 'method': r['method'].encode('utf8'), 'refer': r['refer'].encode('utf8') }) # ----------- 检测网站存活性 by lichao 拿到检测网站存活性的插件id check_ok_vul_id = "" db_session = DBSession() try: vul = db_session.query(WebVulList).filter( WebVulList.script == 'check_web_alive').first() check_ok_vul_id = str(vul.id) except Exception, e: logger.error(e) db_session.close() # ----------- for vulId in scanVulList: from time import time as during_time t1 = during_time() vulId = vulId.replace(" ", "") if vulId == "": continue # ----------- 检测网站存活性 by lichao if not checkOk and len(urlList) <= 1: # 判断网站无法访问 if vulId != check_ok_vul_id: # 网站无法访问时,只运行 check_web_alive 这一个插件 continue else: # 网站可以访问时 if vulId == check_ok_vul_id: # 网站可以访问时,不运行 check_web_alive 这个插件 continue # ------------ progress.append(vulId) self.dao.updateData('sites', {'progress': '|'.join(progress)}, {'id': siteId}) self.dao.deleteData('web_result', { 'vul_id': vulId, 'site_id': siteId }) scanCnf['vulId'] = vulId scanCnf['vulName'] = self.taskCnf['vulDict'][vulId][ 'vul_name'] scanCnf['level'] = self.taskCnf['vulDict'][vulId][ 'level'].encode('utf8') scanCnf['scanType'] = self.taskCnf['vulDict'][vulId][ 'scan_type'] scanCnf['script'] = self.taskCnf['vulDict'][vulId][ 'script'] scanCnf['status'] = '0' scanCnf['endTime'] = time.time() + 1800 scanCnf['timeoutCount'] = 0 #测试爬虫爬出来的路径 if scanCnf['scanType'] == 1: scanCnf['queue'] = Queue() for r in urlList: scanCnf['queue'].put(r) scanUrlScript = ScanScriptForUrl(scanCnf) scanUrlScript.start() #如果只测试指定的URL则不需要运行测试域名和测试漏洞库 if policy != 4: #测试域名 if scanCnf['scanType'] == 2: scanDomainScript = ScanScriptForDomain(scanCnf) scanDomainScript.start() duration = during_time() - t1 # -----------统计插件运行时间 by mcj try: from common.plugin_speed import PluginSpeed db_session = DBSession() plu_speed = PluginSpeed(self.taskId, vulId, duration) db_session.add(plu_speed) db_session.commit() db_session.close() except Exception, e: logger.info(str(e)) db_session.rollback() db_session.close() # -----------统计插件运行时间 by mcj if not checkOk and len(urlList) <= 1: break urlList = []
def init(self): try: if self.action == 'reinit': logger.debug('start to reinit task') reinitDb = {'state':1,'init_state':0,'prescan_state':0,'web_scan_state':0,'weak_pwd_scan_state':0,'port_scan_state':0,'host_scan_state':0,'start_time':'0000-00-00 00:00:00','end_time':'0000-00-00 00:00:00'} self.dao.updateData('task', reinitDb, {'id':self.taskId}) #end if currentTime = time.strftime("%Y-%m-%d %X",time.localtime()) self.dao.updateData('task', {'state':2,'start_time':currentTime,'end_time':'0000-00-00 00:00:00'}, {'id':self.taskId,'state':1}) self.dao.updateData('task', {'state':3,'end_time':currentTime}, {'id':self.taskId,'init_state':1,'prescan_state':1,'web_scan_state':1,'weak_pwd_scan_state':1,'port_scan_state':1,'host_scan_state':1}) taskCnf = self.dao.getTaskData(self.taskId) self.assetTaskId = taskCnf['asset_task_id'] if taskCnf['init_state'] == 1: return #end if target = taskCnf['target'].encode('utf8') if target == '': target = [] else: target = json.read(target.encode('utf8')) #end if #clear host_infos self.dao.deleteData('host_infos', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear host_ports self.dao.deleteData('host_ports', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear sites self.dao.deleteData('sites', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear spider_url self.dao.deleteData('spider_url', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear web_result self.dao.deleteData('web_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear web_result_data self.dao.deleteData('web_result_data', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear host_result self.dao.deleteData('host_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) #clear weak_pwd_result self.dao.deleteData('weak_pwd_result', {'task_id':self.taskId,'asset_task_id':self.assetTaskId}) siteList = [] ipList = [] for item in target: try: domain = '' ip = '' if item.has_key('domain'): domain = item['domain'] #end if if item.has_key('ip'): ip = item['ip'] #end if if domain == '' and ip == '': continue #end if if ip == '': ip = domainToip(domain) if ip == False: continue #end if #end if ipDb = {'task_id':self.taskId,'asset_task_id':self.assetTaskId,'ip':ip} if ip not in ipList: hostId = self.dao.insertData('host_infos', ipDb) if hostId > 0: ipList.append(ip) #end if #end if except Exception, e1: logger.error(e1) #end try #end for self.dao.updateData('task', {'init_state':1}, {'id':self.taskId})