def insertData(self, tablename, data): try: if self.mysqlConnect(): columns = [] values = [] valuesTuple = [] for key in data: columns.append('`' + key + '`') values.append('%s') valuesTuple.append(data[key]) sql = "insert into %s (%s) values (%s)" % ( tablename, ",".join(columns), ",".join(values)) #logger.debug(sql) #logger.debug(tuple(valuesTuple)) self.cursor.execute(sql, tuple(valuesTuple)) self.conn.commit() sql = "select LAST_INSERT_ID() as id" self.cursor.execute(sql) self.conn.commit() res = self.cursor.fetchone() if res and len(res) > 0 and res['id'] > 0: return res['id'] return 0 except Exception, e: logger.exception(e) return 0
def updateSiteException(self, content, siteId, ip): try: exceptionCount = self.dao.getSiteExceptionCount(siteId) if exceptionCount >= 3: siteDb = { 'state': 1, 'exception': '扫描未完成', 'exception_count': int(exceptionCount) + 1, 'next_start_time': datetime.datetime.fromtimestamp(time.time() + 5 * 60) } self.dao.updateData('sites', siteDb, {'id': siteId}) else: siteDb = { 'state': 2, 'exception': '稍后继续尝试', 'exception_count': int(exceptionCount) + 1, 'next_start_time': datetime.datetime.fromtimestamp(time.time() + 5 * 60) } self.dao.updateData('sites', siteDb, {'id': siteId}) self.dao.updateHostWebScanState(self.taskId, self.assetTaskId, ip) except Exception, e: logger.exception(e)
def checkSiteWorkMode(self, res, title): try: if res.has_key('status') and (res['status'] == '404' or res['status'] == '400' or res['status'] == '401'): return False keyword_list = [ 'Internal Server Error', '401 Unauthorized', 'Not Found', 'Bad Request' ] try: temp = u"建设中" keyword_list.append(temp.encode('utf8')) keyword_list.append(temp.encode('gb2312')) temp = u"服务器的使用期限已过" keyword_list.append(temp.encode('utf8')) keyword_list.append(temp.encode('gb2312')) temp = u"网站错误" keyword_list.append(temp.encode('utf8')) keyword_list.append(temp.encode('gb2312')) except Exception, e1: logger.exception(e) for row in keyword_list: try: if title.find(row) >= 0: return False except Exception, e1: continue
def checkExceptionSite(self): flag = False try: self.threadLock.acquire() siteList = self.dao.getData( 'sites', { 'state': '2', 'task_id': self.taskId, 'asset_task_id': self.assetTaskId }) if len(siteList) <= 0: flag = True if siteList: for site in siteList: if site['next_start_time']: if int(time.time()) > int( time.mktime(r['next_start_time'].timetuple())): siteQueue.put(str(site['id'])) self.dao.updateData('sites', { 'state': 0, 'exception': '' }, {'id': str(site['id'])}) except Exception, e: logger.exception(e)
def updateData(self, tablename, data, where): try: if self.mysqlConnect(): values = [] updateSql = '' for k in data: if updateSql == '': updateSql = "`%s` = %s" % (k, '%s') else: updateSql = "%s, `%s` = %s" % (updateSql, k, '%s') values.append(data[k]) whereSql = '' for k in where: if whereSql == '': whereSql = "`%s` = %s" % (k, '%s') else: whereSql = "%s and `%s` = %s" % (whereSql, k, '%s') values.append(where[k]) sql = "update `%s` set %s where %s" % (tablename, updateSql, whereSql) self.cursor.execute(sql, tuple(values)) self.conn.commit() return True return False except Exception, e: logger.exception(e) return False
def PreSiteScan(self, url): try: http = HttpRequest({'timeout': 30, 'follow_redirects': False}) res, content = http.request(url) return True, res, content except socket.timeout, e: logger.exception(e) return False, {}, ''
def updateHostWebScanState(self, taskId, assetTaskId, ip): try: if self.mysqlConnect(): sql = "update host_infos set `web_scan_state` = '1' where (select count(id) from sites where `state` <> '1' and `ip` = '%s' and task_id = '%s' and `asset_task_id` = '%s') = 0 and `ip` = '%s' and task_id = '%s' and `asset_task_id` = '%s'" % ( ip, taskId, assetTaskId, ip, taskId, assetTaskId) self.cursor.execute(sql) self.conn.commit() except Exception, e: logger.exception(e)
def checkSiteId(self, siteId): try: if int(siteId) > 0: return True else: return False except Exception, e: logger.exception(e) return False
def getSiteData(self, siteId, columns='*'): try: if self.mysqlConnect(): sql = "select * from sites where id = '%s'" % (siteId) self.cursor.execute(sql) res = self.cursor.fetchone() if res and len(res) > 0: return res except Exception, e: logger.exception(e)
def getDataBySql(self, sql): try: if self.mysqlConnect(): self.cursor.execute(sql) res = self.cursor.fetchall() if res and len(res) > 0: return res return [] except Exception, e: logger.exception(e) return []
def getUrlList(self, siteId): try: if self.mysqlConnect(): sql = "select * from spider_url where site_id = '%s'" % ( siteId) self.cursor.execute(sql) res = self.cursor.fetchall() if res and len(res) > 0: return res except Exception, e: logger.exception(e)
def mysqlClose(self): try: if self.conn != '' and self.cursor != '': self.cursor.close() self.conn.close() self.conn = '' self.cursor = '' return True except Exception, e: logger.exception(e) return False
def getTaskData(self, taskId, columns='*'): try: if self.mysqlConnect(): sql = "select %s from task where id = %s" % (columns, taskId) self.cursor.execute(sql) res = self.cursor.fetchone() if res and len(res) > 0: return res return False except Exception, e: logger.exception(e) return False
def getSpiderUrlOtherCount(self, siteId=0, url='', params='', method=''): try: if self.mysqlConnect(): sql = "select count(id) as c from spider_url_other where `site_id`=%s and `url`=%s and `params`=%s and `method`=%s" sequence = (siteId, url, params, method) self.cursor.execute(sql, sequence) res = self.cursor.fetchone() if res and len(res): return res['c'] return 0 except Exception, e: logger.exception(e)
def reconnect(self): try: self.conn = MySQLdb.connect(self.host, self.user, self.passwd, db=self.database, charset="utf8") self.cursor = self.conn.cursor(MySQLdb.cursors.DictCursor) return True except Exception, e: logger.exception(e) return False
def cleanWebScan(self, taskId, assetTaskId): try: if self.mysqlConnect(): #清空扫描任务字段 sql = "update `task` set `web_scan_state` = '0', `web_search_site_state` = '0' where `id` = '%s' " % ( taskId) self.cursor.execute(sql) #清空扫描域名记录 if assetTaskId > 0: sql = "delete from `sites` where `task_id` = '%s' and `asset_task_id` = '%s'" % ( taskId, assetTaskId) else: sql = "delete from `sites` where `task_id` = '%s'" % ( taskId) self.cursor.execute(sql) #清空Web扫描结果 if assetTaskId > 0: sql = "delete from `web_result` where `task_id` = '%s' and `asset_task_id` = '%s'" % ( taskId, assetTaskId) else: sql = "delete from `web_result` where `task_id` = '%s'" % ( taskId) self.cursor.execute(sql) #清空Web扫描报文记录 if assetTaskId > 0: sql = "delete from `web_result_data` where `task_id` = '%s' and `asset_task_id` = '%s'" % ( taskId, assetTaskId) else: sql = "delete from `web_result_data` where `task_id` = '%s'" % ( taskId) self.cursor.execute(sql) #清空扫描的URL记录结果 if assetTaskId > 0: sql = "delete from `spider_url` where `task_id` = '%s' and `asset_scan_id` = '%s'" % ( taskId, assetTaskId) else: sql = "delete from `spider_url` where `task_id` = '%s'" % ( taskId) self.cursor.execute(sql) self.conn.commit() return True else: return False except Exception, e: logger.exception(e) return False
def getAssetTaskId(self, taskId): try: if self.mysqlConnect(): sql = "select `asset_task_id` from `task` where `id` = '%s'" % ( taskId) self.cursor.execute(sql) res = self.cursor.fetchone() if res and len(res) > 0: return res['asset_task_id'] return False except Exception, e: logger.exception(e) return False
def getSiteExceptionCount(self, siteId): try: if self.mysqlConnect(): sql = "select `exception_count` from sites where id = '%s'" % ( siteId) self.cursor.execute(sql) res = self.cursor.fetchone() if res and res.has_key('exception_count'): return res['exception_count'] return 0 except Exception, e: logger.exception(e) return 0
def getSiteCode(self, content): try: code = "" match = re.findall(r"<meta(.+?)charset(.*?)=(.+?)(\"|')", content, re.I) if match and len(match) > 0: code = match[0][2] else: code = "utf8" return code except Exception, e: logger.exception(e) return "utf8"
def getWebVulByPolicy(self, policyId): try: if self.mysqlConnect(): vulList = [] sql = "select `vul_id`,`level`,`vul_name`,`scan_type`,`script` from `web_vul_list` where `vul_id` in (select `vul_id` from `web_vul_policy_ref` where `policy_id` = '%s') order by `priority` asc" % ( str(policyId)) self.cursor.execute(sql) res = self.cursor.fetchall() # tuple 转 list 有何意义 mcj if res and len(res) > 0: for row in res: vulList.append(row) return vulList except Exception, e: logger.exception(e)
def ifIpAlive(self, ip): try: try: sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sk.settimeout(self.taskCnf['web_scan_timeout']) sk.connect((ip, 80)) sk.close() return True except Exception, e1: sk.close() logger.exception(e1) return False except Exception, e: logger.exception(e) return False
def __init__(self, taskId, assetTaskId, taskCnf, threadLock): logger.debug("__init__ scansite") try: threading.Thread.__init__(self) self.module = self.__class__.__name__ self.taskId = taskId self.assetTaskId = assetTaskId self.taskCnf = taskCnf self.threadLock = threadLock self.threadName = threading.currentThread().getName() self.dao = MysqlDao() self.count = 0 except Exception, e: logger.exception(e)
def __init__(self, taskId, taskCnf): try: threading.Thread.__init__(self) self.module = self.__class__.__name__ self.taskId = taskId self.assetTaskId = taskCnf['asset_task_id'] self.taskCnf = taskCnf self.sitePorts = [80, 81, 443, 8080] self.http = HttpRequest( {'timeout': self.taskCnf['web_search_site_timeout']}) self.htmlParser = HTMLParser.HTMLParser() self.ipList = [] self.dao = MysqlDao() except Exception, e: logger.exception(e)
def getRowData(self, tablename, where, columns='*'): try: if self.mysqlConnect(): whereSql = ' 1 = 1 ' values = [] for k in where: whereSql = "%s and `%s` = %s" % (whereSql, k, '%s') values.append(where[k]) sql = "select %s from %s where %s" % (columns, tablename, whereSql) self.cursor.execute(sql, tuple(values)) res = self.cursor.fetchone() if res and len(res) > 0: return res except Exception, e: logger.exception(e)
def checkErrorFileStatus(self, scheme, domain, path, type, method): try: http = httplib2.Http(disable_ssl_certificate_validation=True) http.follow_redirects = False socket.setdefaulttimeout(30) url = "%s://%s%snulllllllllll%s" % (scheme, domain, path, type) if method.lower() == "head": res, content = http.request(url, "HEAD") else: res, content = http.request(url) if res and res.has_key('status') and res['status'] == '404': return True else: return False except Exception, e: logger.exception(e) return False
def __init__(self, host='', database='', user='', passwd=''): try: if host == '' or database == '' or user == '' or passwd == '': self.host = SCANER_DB_HOST self.database = SCANER_DB_DATABASE self.user = SCANER_DB_USER self.passwd = SCANER_DB_PASSWORD else: self.host = host self.database = database self.user = user self.passwd = passwd self.conn = '' self.cursor = '' self.module = self.__class__.__name__ except Exception, e: logger.exception(e)
def getUnscandSite(self, taskId, assetTaskId=0): try: siteIds = [] sql = "select `id` from `sites` where `state` <> '1' and `task_id` = '%s' and `asset_task_id` = '%s'" % ( taskId, assetTaskId) if self.mysqlConnect(): self.cursor.execute(sql) self.conn.commit() res = self.cursor.fetchall() if res and len(res) > 0: for row in res: siteIds.append(str(row['id'])) return siteIds except Exception, e: logger.exception(e) return []
def checkHeadRequest(self, scheme, domain, path): try: http = httplib2.Http(disable_ssl_certificate_validation=True) http.follow_redirects = False socket.setdefaulttimeout(30) url = "%s://%s%s" % (scheme, domain, path) res, content = http.request(url, "HEAD") if res and res.has_key('status') and res['status'] in [ '200', '301', '302', '403' ] and res.has_key('content-length'): return True else: return False except Exception, e: logger.exception(e) return False
def searchOtherSiteInIp(self, ip): try: getSiteByIpUrl = 'http://test.com/dns?ip=%s' % (ip) res, content = self.http.request(url) if content == '': return False content = json.read(content) if isinstance(content, list) == False: return False for row in content: siteObj = { 'scheme': 'http', 'domain': '', 'path': '/', 'ip': ip, 'title': '', 'policy': 1, 'include_url': '', 'exclude_url': '', 'cookie': '', 'sub_domain_scan': 0, 'ip_domain_scan': 0 } if row.has_key('domain') == False: continue siteObj['domain'] = row['domain'] if row.has_key('scheme'): siteObj['scheme'] = row['scheme'] if row.has_key('path'): siteObj['path'] = row['path'] if row.has_key('title'): siteObj['title'] = row['title'] self.updateTaskSites(siteObj) return True except Exception, e: logger.exception(e)
def deleteData(self, tablename, where): try: if self.mysqlConnect(): values = [] whereSql = '' for k in where: if whereSql == '': whereSql = "`%s` = %s" % (k, '%s') else: whereSql = "%s and `%s` = %s" % (whereSql, k, '%s') values.append(where[k]) sql = "delete from %s where %s" % (tablename, whereSql) self.cursor.execute(sql, tuple(values)) self.conn.commit() return True except Exception, e: logger.exception(e)