def insertData(self, tablename, data):
try:
if self.mysqlConnect():
columns = []
values = []
valuesTuple = []
for key in data:
columns.append('`' + key + '`')
values.append('%s')
valuesTuple.append(data[key])
sql = "insert into %s (%s) values (%s)" % (
tablename, ",".join(columns), ",".join(values))
#logger.debug(sql)
#logger.debug(tuple(valuesTuple))
self.cursor.execute(sql, tuple(valuesTuple))
self.conn.commit()
sql = "select LAST_INSERT_ID() as id"
self.cursor.execute(sql)
self.conn.commit()
res = self.cursor.fetchone()
if res and len(res) > 0 and res['id'] > 0:
return res['id']
return 0
except Exception, e:
logger.exception(e)
return 0
def updateSiteException(self, content, siteId, ip):
try:
exceptionCount = self.dao.getSiteExceptionCount(siteId)
if exceptionCount >= 3:
siteDb = {
'state':
1,
'exception':
'扫描未完成',
'exception_count':
int(exceptionCount) + 1,
'next_start_time':
datetime.datetime.fromtimestamp(time.time() + 5 * 60)
}
self.dao.updateData('sites', siteDb, {'id': siteId})
else:
siteDb = {
'state':
2,
'exception':
'稍后继续尝试',
'exception_count':
int(exceptionCount) + 1,
'next_start_time':
datetime.datetime.fromtimestamp(time.time() + 5 * 60)
}
self.dao.updateData('sites', siteDb, {'id': siteId})
self.dao.updateHostWebScanState(self.taskId, self.assetTaskId, ip)
except Exception, e:
logger.exception(e)
def checkSiteWorkMode(self, res, title):
try:
if res.has_key('status') and (res['status'] == '404'
or res['status'] == '400'
or res['status'] == '401'):
return False
keyword_list = [
'Internal Server Error', '401 Unauthorized', 'Not Found',
'Bad Request'
]
try:
temp = u"建设中"
keyword_list.append(temp.encode('utf8'))
keyword_list.append(temp.encode('gb2312'))
temp = u"服务器的使用期限已过"
keyword_list.append(temp.encode('utf8'))
keyword_list.append(temp.encode('gb2312'))
temp = u"网站错误"
keyword_list.append(temp.encode('utf8'))
keyword_list.append(temp.encode('gb2312'))
except Exception, e1:
logger.exception(e)
for row in keyword_list:
try:
if title.find(row) >= 0:
return False
except Exception, e1:
continue
def checkExceptionSite(self):
flag = False
try:
self.threadLock.acquire()
siteList = self.dao.getData(
'sites', {
'state': '2',
'task_id': self.taskId,
'asset_task_id': self.assetTaskId
})
if len(siteList) <= 0:
flag = True
if siteList:
for site in siteList:
if site['next_start_time']:
if int(time.time()) > int(
time.mktime(r['next_start_time'].timetuple())):
siteQueue.put(str(site['id']))
self.dao.updateData('sites', {
'state': 0,
'exception': ''
}, {'id': str(site['id'])})
except Exception, e:
logger.exception(e)
def updateData(self, tablename, data, where):
try:
if self.mysqlConnect():
values = []
updateSql = ''
for k in data:
if updateSql == '':
updateSql = "`%s` = %s" % (k, '%s')
else:
updateSql = "%s, `%s` = %s" % (updateSql, k, '%s')
values.append(data[k])
whereSql = ''
for k in where:
if whereSql == '':
whereSql = "`%s` = %s" % (k, '%s')
else:
whereSql = "%s and `%s` = %s" % (whereSql, k, '%s')
values.append(where[k])
sql = "update `%s` set %s where %s" % (tablename, updateSql,
whereSql)
self.cursor.execute(sql, tuple(values))
self.conn.commit()
return True
return False
except Exception, e:
logger.exception(e)
return False
def PreSiteScan(self, url):
try:
http = HttpRequest({'timeout': 30, 'follow_redirects': False})
res, content = http.request(url)
return True, res, content
except socket.timeout, e:
logger.exception(e)
return False, {}, ''
def updateHostWebScanState(self, taskId, assetTaskId, ip):
try:
if self.mysqlConnect():
sql = "update host_infos set `web_scan_state` = '1' where (select count(id) from sites where `state` <> '1' and `ip` = '%s' and task_id = '%s' and `asset_task_id` = '%s') = 0 and `ip` = '%s' and task_id = '%s' and `asset_task_id` = '%s'" % (
ip, taskId, assetTaskId, ip, taskId, assetTaskId)
self.cursor.execute(sql)
self.conn.commit()
except Exception, e:
logger.exception(e)
def checkSiteId(self, siteId):
try:
if int(siteId) > 0:
return True
else:
return False
except Exception, e:
logger.exception(e)
return False
def getSiteData(self, siteId, columns='*'):
try:
if self.mysqlConnect():
sql = "select * from sites where id = '%s'" % (siteId)
self.cursor.execute(sql)
res = self.cursor.fetchone()
if res and len(res) > 0:
return res
except Exception, e:
logger.exception(e)
def getDataBySql(self, sql):
try:
if self.mysqlConnect():
self.cursor.execute(sql)
res = self.cursor.fetchall()
if res and len(res) > 0:
return res
return []
except Exception, e:
logger.exception(e)
return []
def getUrlList(self, siteId):
try:
if self.mysqlConnect():
sql = "select * from spider_url where site_id = '%s'" % (
siteId)
self.cursor.execute(sql)
res = self.cursor.fetchall()
if res and len(res) > 0:
return res
except Exception, e:
logger.exception(e)
def mysqlClose(self):
try:
if self.conn != '' and self.cursor != '':
self.cursor.close()
self.conn.close()
self.conn = ''
self.cursor = ''
return True
except Exception, e:
logger.exception(e)
return False
def getTaskData(self, taskId, columns='*'):
try:
if self.mysqlConnect():
sql = "select %s from task where id = %s" % (columns, taskId)
self.cursor.execute(sql)
res = self.cursor.fetchone()
if res and len(res) > 0:
return res
return False
except Exception, e:
logger.exception(e)
return False
def getSpiderUrlOtherCount(self, siteId=0, url='', params='', method=''):
try:
if self.mysqlConnect():
sql = "select count(id) as c from spider_url_other where `site_id`=%s and `url`=%s and `params`=%s and `method`=%s"
sequence = (siteId, url, params, method)
self.cursor.execute(sql, sequence)
res = self.cursor.fetchone()
if res and len(res):
return res['c']
return 0
except Exception, e:
logger.exception(e)
def reconnect(self):
try:
self.conn = MySQLdb.connect(self.host,
self.user,
self.passwd,
db=self.database,
charset="utf8")
self.cursor = self.conn.cursor(MySQLdb.cursors.DictCursor)
return True
except Exception, e:
logger.exception(e)
return False
def cleanWebScan(self, taskId, assetTaskId):
try:
if self.mysqlConnect():
#清空扫描任务字段
sql = "update `task` set `web_scan_state` = '0', `web_search_site_state` = '0' where `id` = '%s' " % (
taskId)
self.cursor.execute(sql)
#清空扫描域名记录
if assetTaskId > 0:
sql = "delete from `sites` where `task_id` = '%s' and `asset_task_id` = '%s'" % (
taskId, assetTaskId)
else:
sql = "delete from `sites` where `task_id` = '%s'" % (
taskId)
self.cursor.execute(sql)
#清空Web扫描结果
if assetTaskId > 0:
sql = "delete from `web_result` where `task_id` = '%s' and `asset_task_id` = '%s'" % (
taskId, assetTaskId)
else:
sql = "delete from `web_result` where `task_id` = '%s'" % (
taskId)
self.cursor.execute(sql)
#清空Web扫描报文记录
if assetTaskId > 0:
sql = "delete from `web_result_data` where `task_id` = '%s' and `asset_task_id` = '%s'" % (
taskId, assetTaskId)
else:
sql = "delete from `web_result_data` where `task_id` = '%s'" % (
taskId)
self.cursor.execute(sql)
#清空扫描的URL记录结果
if assetTaskId > 0:
sql = "delete from `spider_url` where `task_id` = '%s' and `asset_scan_id` = '%s'" % (
taskId, assetTaskId)
else:
sql = "delete from `spider_url` where `task_id` = '%s'" % (
taskId)
self.cursor.execute(sql)
self.conn.commit()
return True
else:
return False
except Exception, e:
logger.exception(e)
return False
def getAssetTaskId(self, taskId):
try:
if self.mysqlConnect():
sql = "select `asset_task_id` from `task` where `id` = '%s'" % (
taskId)
self.cursor.execute(sql)
res = self.cursor.fetchone()
if res and len(res) > 0:
return res['asset_task_id']
return False
except Exception, e:
logger.exception(e)
return False
def getSiteExceptionCount(self, siteId):
try:
if self.mysqlConnect():
sql = "select `exception_count` from sites where id = '%s'" % (
siteId)
self.cursor.execute(sql)
res = self.cursor.fetchone()
if res and res.has_key('exception_count'):
return res['exception_count']
return 0
except Exception, e:
logger.exception(e)
return 0
def getSiteCode(self, content):
try:
code = ""
match = re.findall(r"<meta(.+?)charset(.*?)=(.+?)(\"|')", content,
re.I)
if match and len(match) > 0:
code = match[0][2]
else:
code = "utf8"
return code
except Exception, e:
logger.exception(e)
return "utf8"
def getWebVulByPolicy(self, policyId):
try:
if self.mysqlConnect():
vulList = []
sql = "select `vul_id`,`level`,`vul_name`,`scan_type`,`script` from `web_vul_list` where `vul_id` in (select `vul_id` from `web_vul_policy_ref` where `policy_id` = '%s') order by `priority` asc" % (
str(policyId))
self.cursor.execute(sql)
res = self.cursor.fetchall()
# tuple 转 list 有何意义 mcj
if res and len(res) > 0:
for row in res:
vulList.append(row)
return vulList
except Exception, e:
logger.exception(e)
def ifIpAlive(self, ip):
try:
try:
sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sk.settimeout(self.taskCnf['web_scan_timeout'])
sk.connect((ip, 80))
sk.close()
return True
except Exception, e1:
sk.close()
logger.exception(e1)
return False
except Exception, e:
logger.exception(e)
return False
def __init__(self, taskId, assetTaskId, taskCnf, threadLock):
logger.debug("__init__ scansite")
try:
threading.Thread.__init__(self)
self.module = self.__class__.__name__
self.taskId = taskId
self.assetTaskId = assetTaskId
self.taskCnf = taskCnf
self.threadLock = threadLock
self.threadName = threading.currentThread().getName()
self.dao = MysqlDao()
self.count = 0
except Exception, e:
logger.exception(e)
def __init__(self, taskId, taskCnf):
try:
threading.Thread.__init__(self)
self.module = self.__class__.__name__
self.taskId = taskId
self.assetTaskId = taskCnf['asset_task_id']
self.taskCnf = taskCnf
self.sitePorts = [80, 81, 443, 8080]
self.http = HttpRequest(
{'timeout': self.taskCnf['web_search_site_timeout']})
self.htmlParser = HTMLParser.HTMLParser()
self.ipList = []
self.dao = MysqlDao()
except Exception, e:
logger.exception(e)
def getRowData(self, tablename, where, columns='*'):
try:
if self.mysqlConnect():
whereSql = ' 1 = 1 '
values = []
for k in where:
whereSql = "%s and `%s` = %s" % (whereSql, k, '%s')
values.append(where[k])
sql = "select %s from %s where %s" % (columns, tablename,
whereSql)
self.cursor.execute(sql, tuple(values))
res = self.cursor.fetchone()
if res and len(res) > 0:
return res
except Exception, e:
logger.exception(e)
def checkErrorFileStatus(self, scheme, domain, path, type, method):
try:
http = httplib2.Http(disable_ssl_certificate_validation=True)
http.follow_redirects = False
socket.setdefaulttimeout(30)
url = "%s://%s%snulllllllllll%s" % (scheme, domain, path, type)
if method.lower() == "head":
res, content = http.request(url, "HEAD")
else:
res, content = http.request(url)
if res and res.has_key('status') and res['status'] == '404':
return True
else:
return False
except Exception, e:
logger.exception(e)
return False
def __init__(self, host='', database='', user='', passwd=''):
try:
if host == '' or database == '' or user == '' or passwd == '':
self.host = SCANER_DB_HOST
self.database = SCANER_DB_DATABASE
self.user = SCANER_DB_USER
self.passwd = SCANER_DB_PASSWORD
else:
self.host = host
self.database = database
self.user = user
self.passwd = passwd
self.conn = ''
self.cursor = ''
self.module = self.__class__.__name__
except Exception, e:
logger.exception(e)
def getUnscandSite(self, taskId, assetTaskId=0):
try:
siteIds = []
sql = "select `id` from `sites` where `state` <> '1' and `task_id` = '%s' and `asset_task_id` = '%s'" % (
taskId, assetTaskId)
if self.mysqlConnect():
self.cursor.execute(sql)
self.conn.commit()
res = self.cursor.fetchall()
if res and len(res) > 0:
for row in res:
siteIds.append(str(row['id']))
return siteIds
except Exception, e:
logger.exception(e)
return []
def checkHeadRequest(self, scheme, domain, path):
try:
http = httplib2.Http(disable_ssl_certificate_validation=True)
http.follow_redirects = False
socket.setdefaulttimeout(30)
url = "%s://%s%s" % (scheme, domain, path)
res, content = http.request(url, "HEAD")
if res and res.has_key('status') and res['status'] in [
'200', '301', '302', '403'
] and res.has_key('content-length'):
return True
else:
return False
except Exception, e:
logger.exception(e)
return False
def searchOtherSiteInIp(self, ip):
try:
getSiteByIpUrl = 'http://test.com/dns?ip=%s' % (ip)
res, content = self.http.request(url)
if content == '':
return False
content = json.read(content)
if isinstance(content, list) == False:
return False
for row in content:
siteObj = {
'scheme': 'http',
'domain': '',
'path': '/',
'ip': ip,
'title': '',
'policy': 1,
'include_url': '',
'exclude_url': '',
'cookie': '',
'sub_domain_scan': 0,
'ip_domain_scan': 0
}
if row.has_key('domain') == False:
continue
siteObj['domain'] = row['domain']
if row.has_key('scheme'):
siteObj['scheme'] = row['scheme']
if row.has_key('path'):
siteObj['path'] = row['path']
if row.has_key('title'):
siteObj['title'] = row['title']
self.updateTaskSites(siteObj)
return True
except Exception, e:
logger.exception(e)
def deleteData(self, tablename, where):
try:
if self.mysqlConnect():
values = []
whereSql = ''
for k in where:
if whereSql == '':
whereSql = "`%s` = %s" % (k, '%s')
else:
whereSql = "%s and `%s` = %s" % (whereSql, k, '%s')
values.append(where[k])
sql = "delete from %s where %s" % (tablename, whereSql)
self.cursor.execute(sql, tuple(values))
self.conn.commit()
return True
except Exception, e:
logger.exception(e)
