def writedb(data):
try:
config = getinfo(filename)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
# sql = 'desc ids_info'
attacktype = data[u'attack_type']
hostname = data[u'hostname']
status = int(data[u'status'])
method = data[u'method']
url = data[u'url']
baseurl = base64.b64encode(url)
if method.lower() == 'post':
post = data[u'post']
basepost = base64.b64encode(post)
# write to db
if method.lower() == 'get':
insertsql1 = 'insert into ids_info(attack_type, hostname, status, method, url ) ' \
'values("%s", "%s", %d, "%s", "%s")' % (attacktype, hostname, status, method, baseurl)
cur.execute(insertsql1)
conn.commit()
elif method.lower() == 'post':
insertsql2 = 'insert into ids_info(attack_type, hostname, status, method, url, postdata ) ' \
'values("%s", "%s", %d, "%s", "%s", "%s")' % (attacktype, hostname, status, method, baseurl, basepost)
# print insertsql2
cur.execute(insertsql2)
conn.commit()
else:
print 'what?'
# result_set = cur.fetchall()
# print result_set
conn.close()
except Exception as e:
record_err.logrecord()
def deldupl(idsdata):
try:
duplicatedfile = 'logs/duplicate_attack.txt'
duplicateother = 'logs/duplicate_otheratt.txt'
dupedfile = open(duplicatedfile, 'a')
# load list
# print load_write_listfile.checkfile(), here get a large list!
attackedlist = sur_load_listfile.checkfile()
# filereadlist, xsslist, sqlilist, cvelist, iislist, urlscanlist, cmdexelist, scanlist, xxelist = \
# sur_load_listfile.checkfile()
count = 0
# tmp means hostname and url, join the new url+host and match if it is in the loaded list
if 'hostname' in idsdata.keys() and 'url' in idsdata.keys():
tmp = idsdata['hostname'] + idsdata['url']
if tmp not in attackedlist:
attackedlist.append(tmp)
dupedfile.write(str(idsdata) + '\n')
dupedfile.close()
sur_load_listfile.writelist(attackedlist)
else:
dupofile = open(duplicateother, 'a')
dupofile.write(str(idsdata) + '\n')
dupofile.close()
except Exception as e:
record_err.logrecord()
def readfile(startid=1):
try:
id = startid
config = getinfo(filename)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
readsql = 'select attack_type, hostname, url, method, status, postdata from ids_info where id = %d' % startid
cur.execute(readsql)
# get info
result = cur.fetchall()
if len(result) > 0:
attack_type = result[0][0]
hostname = result[0][1]
url = result[0][2]
method = result[0][3]
status = result[0][4]
postdata = result[0][5]
else:
print 'mysql info error'
if attack_type == u'文件读取':
catfileread(id, hostname, url, method, status, postdata)
else:
# process other attack_type
pass
except Exception as e:
record_err.logrecord()
def filecount(logfile):
try:
stats, output = commands.getstatusoutput('wc -l %s' % (logfile))
count = int(output.split()[0])
return count
except Exception as e:
record_err.logrecord()
def count(self):
#返回文件行数
try:
status, output = commands.getstatusoutput('wc -l %s' %self.filepath)
output = output.split()
return int(output[0])
except Exception as e:
record_err.logrecord()
def writelist(attackedlist):
"""
write the new list to file!
"""
try:
file1 = open(filelist, 'w')
file1.write(str(attackedlist))
except Exception as e:
record_err.logrecord()
def content_process(id, content):
try:
if len(content) < 10:
result = 'N'
return id, result
else:
print content
except Exception as e:
record_err.logrecord()
def readfile(content):
# change str to dict
try:
# print content, type(content)
dict_content = eval(content)
if dict_content[u'hostname'] != u'www.jiedaibao.com':
read_file_mysql_delsame.catagory(dict_content)
except Exception as e:
record_err.logrecord()
def test():
try:
print 'hello'
print '似懂非懂分sdfdfdf' \
'sabc', \
'sdf'
content = ('hello sdfsdf' '2sdf')
print content
except Exception as e:
record_err.logrecord()
test2()
def writedb(data):
try:
#"注意一些字段担心有特殊字符无法写入db，进行了base64编码"
data = eval(data)
config = getinfo(myfile)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
# sql = 'desc ids_info'
# url, useragent, postdata, need base64code
status = data['status']
catagory = data['catagory']
alert = data['alert']
postdata = base64.b64encode(data['postdata'])
url = base64.b64encode(data['url'])
hostname = base64.b64encode(data['hostname'])
# datetime = data['datetime']
method = data['method']
# srcip = data['src_ip']
# srcport = data['src_port']
# dstip = data['dest_ip']
dstport = data['dest_port']
length = data['length']
if 'useragent' in data.keys():
useragent = base64.b64encode(data['useragent'])
else:
useragent = ''
if 'xff' in data.keys():
xff = data['xff']
else:
xff = ''
if 'payload' in data.keys():
payload = data['payload']
else:
payload = ''
insertsql = 'insert into httpattack(catagory, dstport, hostname, url, method, length, useragent, \
postdata, payload, status) values("%s" ,"%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")' \
%(catagory, dstport, hostname, url, method, length, useragent, postdata, payload, status)
# print insertsql
cur.execute(insertsql)
conn.commit()
# elif method.lower() == 'post':
# insertsql2 = 'insert into ids_info(attack_type, hostname, status, method, url, postdata ) ' \
# 'values("%s", "%s", %d, "%s", "%s", "%s")' % (attacktype, hostname, status, method, baseurl, basepost)
# # print insertsql2
# cur.execute(insertsql2)
# conn.commit()
# else:
# print 'what?'
# # result_set = cur.fetchall()
# # print result_set
conn.close()
except Exception as e:
record_err.logrecord()
def filename(logfile, file_count, startline):
try:
# 控制读取次数
for i in range(file_count - startline + 1):
content = linecache.getline(logfile, startline)
readfile(content)
startline += 1
# 已处理到多少行
linecache.clearcache()
return startline - 1
except Exception as e:
record_err.logrecord()
def loadlist(filelist):
try:
attackedlist = []
# print filelist
file1 = open(filelist)
content = file1.readlines()
if len(content) != 0:
tmp = content[0]
attackedlist = eval(tmp)
return attackedlist
except Exception as e:
record_err.logrecord()
def getinfo(filename):
try:
file = open(filename, 'r')
config = {}
content = file.readlines()
for i in range(len(content)):
tmp = content[i].split(':')
key = tmp[0]
value = tmp[1].split('\n')[0]
config[key] = value
return config
except Exception as e:
record_err.logrecord()
def getcount():
try:
# get msyql line number
config = getinfo(myconf)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
linecount = 'select count(id) from httpattack '
cur.execute(linecount)
result = cur.fetchall()
return int(result[0][0])
cur.close()
conn.close()
except Exception as e:
record_err.logrecord()
def check(id, hostname, url, method, status, postdata):
try:
result1 = ''
result2 = ''
# print id, hostname, url, method, status, postdata
# print type(id), type(hostname), type(url), type(method), type(status), type(postdata)
url = base64.b64decode(url)
headers = {'user-agent': 'Chrome/60.0.3112.113 Safarids24/537.36'}
if method.lower() == 'post':
postdata = base64.b64decode(postdata)
# print postdata
if method.lower() == 'get':
httpurl = 'http://' + hostname + url
httpsurl = 'https://' + hostname + url
try:
r1 = requests.get(httpurl, headers=headers)
except:
r1 = ''
if r1 is not '':
tmp1 = str(r1)
tmp2 = tmp1.split()
httpstatus = tmp2[1]
if httpstatus:
if re.search('200', httpstatus):
httpcontent = r1.text
id, result1 = content_process(id, httpcontent)
try:
r2 = requests.get(httpsurl, headers=headers, verify=False)
except:
r2 = ''
if r2 is not '':
print r2
result = ''
tmp3 = str(r2)
tmp4 = tmp3.split()
httpsstatus = tmp4[1]
if httpsstatus:
if re.search('200', httpsstatus):
httpscontent = r2.text
id, result2 = content_process(id, httpscontent)
if result1 is not '':
print id, result1
elif result2 is not '':
print id, result2
elif result1 is '' and result2 is '':
print id, 'cannot open!'
else:
pass
except Exception as e:
record_err.logrecord()
def checkfile():
try:
count = 0
filecount = len(filelist)
for i in range(len(filelist)):
result = os.path.exists(filelist)
if result is False:
commands.getstatusoutput('touch %s' % filelist)
result = os.path.exists(filelist)
if result is True:
count += 1
elif result is True:
count += 1
if count == filecount:
return loadlist(filelist)
except Exception as e:
record_err.logrecord()
def fileinfo(logfile, start_line=1):
try:
# 获取文件行数
file_count = filecount(logfile)
processed_line = filename(logfile, file_count, start_line)
while 1:
file_record = open('/tmp/file_no.txt', 'a')
file_count = filecount(logfile)
if file_count > processed_line:
processed_line = filename(logfile, file_count,
processed_line + 1)
# record the last processed line number
file_record.write(str(processed_line))
file_record.close()
time.sleep(3)
except Exception as e:
record_err.logrecord()
def getuntestline():
try:
# get the untest id
count = getcount()
for i in range(count):
config = getinfo(myconf)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
readsql = 'select id from httpattack where attack_status is NULL limit %d,1' % i
cur.execute(readsql)
result = cur.fetchall()
id = result[0][0]
# return id
cur.close()
conn.close()
readfile(id)
except Exception as e:
record_err.logrecord()
def catagory(attack_data):
try:
duplicatedfile = '/tmp/duplicate_attack.txt'
dupfile = open(duplicatedfile, 'a')
# load list
# print load_write_listfile.checkfile()
filereadlist, xsslist, sqlilist = load_write_listfile.checkfile()
count = 0
# tmp means hostname and url, join the new url+host and match if it is in the loaded list
tmp = attack_data[u'url'] + attack_data[u'hostname']
if attack_data[u'attack_type'] == u'文件读取' and attack_data[
u'status'] == 200:
# global filereadlist
if tmp not in filereadlist:
filereadlist.append(tmp)
write_todb.writedb(attack_data)
dupfile.write(str(attack_data))
count += 1
# print attack_data
elif attack_data[u'attack_type'] == u'XSS攻击' and attack_data[
u'status'] == 200:
# global xsslist
if tmp not in xsslist:
xsslist.append(tmp)
write_todb.writedb(attack_data)
dupfile.write(str(attack_data))
count += 1
elif attack_data[u'attack_type'] == u'SQL注入':
# global sqlinject
if tmp not in sqlilist:
sqlilist.append(tmp)
write_todb.writedb(attack_data)
dupfile.write(str(attack_data))
count += 1
else:
pass
if count > 0:
# write the new list to file
dupfile.close()
load_write_listfile.writelist(filereadlist, xsslist, sqlilist)
except Exception as e:
record_err.logrecord()
def readfile(content):
# change str to dict
try:
new_dict = {}
# if content is json str, convert to dict
con_dict = json.loads(content)
# get http attack type and info
if con_dict[u'subproto']:
if con_dict[u'subproto'] == 'http':
new_dict[u'attack_type'] = con_dict[u'attack_type']
new_dict[u'hostname'] = con_dict[u'hostname']
new_dict[u'url'] = con_dict[u'url']
new_dict[u'method'] = con_dict[u'method']
new_dict[u'status'] = con_dict[u'status']
if con_dict[u'method'] == 'POST':
new_dict[u'post'] = con_dict[u'postdata']
attack_deliver.catagory(new_dict)
# else , pass
except Exception as e:
record_err.logrecord()
def readfile(startid = 1):
try:
id = startid
config = getinfo(myconf)
conn = mysql.connector.connect(**config)
cur = conn.cursor()
readsql = 'select catagory, hostname, url, method, status, postdata, payload, dstport from httpattack where id = %d' % startid
cur.execute(readsql)
# get info
result = cur.fetchall()
if len(result) > 0:
# "get data from db is unicode"
catagory = str(result[0][0])
hostname = str(base64.b64decode(result[0][1]))
url = str(base64.b64decode(result[0][2]))
method = str(result[0][3])
status = str(result[0][4])
postdata = str(result[0][5])
payload = str(result[0][6])
dstport = str(result[0][7])
if len(payload) > 0:
payload = str(base64.b64decode(payload))
if method.lower() == 'post':
if len(postdata) > 0:
postdata = str(base64.b64decode(postdata))
else:
print 'mysql info error'
if catagory == u'read_file':
#需要判断是否需要登录
# print hostname, url, dstport, method, postdata, status
check_fileread_attack.checkstatus(id, hostname, url, method, status, postdata)
# catfileread(id, hostname, url, method, status, postdata)
else:
# process other attack_type
pass
except Exception as e:
record_err.logrecord()
class Attack(object):
try:
def __init__(self, data):
self.data = data
self.data['result'] = ''
# 结果有攻击成功(succesd)、攻击失败(failed)、未知(unknown)
def whitecheck(self):
file = open('conf/whitelist.txt', 'r')
rule = eval(file.readline())
if 'hostname' in self.data.keys():
if self.data['hostname'] in rule['whitelist']:
self.data['result'] = 'uncheck'
return self.data
else:
return 0
else:
return 0
# print type(self.data), self.data
def result(self):
pass
#返回检测结果
def statuscheck(self):
#状态检查，如果返回1则代表返回非200，否则返回0，没有status或者返回200
#1 代表非200等，页面无法正常返回，一般代表页面无法打开，0代表需要进一步测试
if 'status' in self.data.keys():
#判断status是否为空值
if self.data['status'] != '':
if self.data['status'] < 200 or self.data['status'] > 299:
return 1
else:
return 0
else:
return 0
else:
return 0
def scancheck(self):
pass
def cvecheck(self):
pass
def iischeck(self):
pass
def xsscheck(self):
pass
def readfile(self):
status = self.statuscheck()
if status == 1:
self.data['result'] = 'failed'
return self.data
else:
#这里需要一个白名单，检测返回结果，在白名单里面的失败
url = self.data['url']
url = url.replace('\/', '/')
#因为里面有转译字符，需要将转译字符取消
header = {
'user-agent': 'Chrome/60.0.3112.113 Safarids24/537.36'
}
hostname = self.data['hostname']
httpurl = 'http://' + hostname + url
httpsurl = 'https://' + hostname + url
#需要进行实际测试，先判断方法
if self.data['method'].lower() == 'get':
r1 = requests.get(httpurl, headers=header)
r2 = requests.get(httpsurl, headers=header, verify=False)
if r1.content.find('请求异常') >= 0:
print 'failed'
elif r1.content.find('系统错误') >= 0:
print 'failed'
elif r1.content.find('Bad Request') >= 0:
print 'failed'
else:
print r1.content
elif self.data['method'].lower() == 'post':
pass
else:
pass
print self.data['method']
self.data['result'] = 'uncheck'
#return self.data
except Exception as e:
record_err.logrecord()
pass
elif newcontent['attacktype'] == u'url扫描':
pass
elif newcontent['attacktype'] == u'命令执行':
pass
elif newcontent['attacktype'] == u'文件读取':
checkresult = attack.readfile()
elif newcontent['attacktype'] == u'sql注入':
pass
elif newcontent['attacktype'] == u'url扫描':
pass
else:
pass
else:
#这里代表检测完毕，可以输出到文本了
pass
# print checkresult
# Attackengine.Attack(newcontent)
except:
log.write(str(fromline) + 'line is finished!\n')
continue
log.close()
else:
print fromline
time.sleep(10)
readfile = File(syspath, fromline)
filelines = readfile.count()
except Exception as e:
record_err.logrecord()
def readfile(content):
# change str to dict
try:
#"otherattackfile 记录的是非http流量"
otherattackfile = open('logs/other_attack.txt', 'a')
new_dict = {}
new_dict['status'] = ''
new_dict['postdata'] = ''
new_dict['payload'] = ''
new_dict['method'] = ''
new_dict['length'] = ''
# if content is json str, convert to dict
con_dict = json.loads(content)
# otherattackfile.write(str(con_dict))
# otherattackfile.write('\n')
# check if content is http or not
conkey = con_dict.keys()
if u'http' in conkey:
new_dict['http'] = (con_dict[u'http'])
if u'status' in new_dict['http'].keys():
new_dict['status'] = str(new_dict[u'http'][u'status'])
if u'http_user_agent' in new_dict['http'].keys():
new_dict['useragent'] = str(
new_dict[u'http'][u'http_user_agent'])
if u'url' in new_dict['http'].keys():
new_dict['url'] = str(new_dict[u'http'][u'url'])
if u'hostname' in new_dict['http'].keys():
new_dict['hostname'] = str(new_dict[u'http'][u'hostname'])
if u'xff' in new_dict['http'].keys():
new_dict['xff'] = str(new_dict[u'http'][u'xff'])
if u'http_method' in new_dict['http'].keys():
new_dict['method'] = str(new_dict[u'http'][u'http_method'])
if u'request_body' in new_dict['http'].keys():
new_dict['postdata'] = str(new_dict[u'http'][u'request_body'])
if u'length' in new_dict['http'].keys():
new_dict['length'] = str(new_dict[u'http'][u'length'])
if u'src_ip' in conkey:
new_dict['src_ip'] = str(con_dict[u'src_ip'])
if u'src_port' in conkey:
new_dict['src_port'] = str(con_dict[u'src_port'])
if u'dest_ip' in conkey:
new_dict['dest_ip'] = str(con_dict[u'dest_ip'])
if u'dest_port' in conkey:
new_dict['dest_port'] = str(con_dict[u'dest_port'])
if u'timestamp' in conkey:
new_dict['datetime'] = str(con_dict[u'timestamp'])
if u'alert' in conkey:
new_dict['alert'] = str(con_dict[u'alert'][u'signature'])
if u'payload' in conkey:
new_dict['payload'] = str(con_dict[u'payload'])
new_dict.pop('http')
# debug test
catagory(new_dict)
# debug leave @ 20170930
# httpattackfile.write(str(con_dict))
# httpattackfile.write('\n')
# httpattackfile.close()
else:
pass
# print con_dict
otherattackfile.write(str(con_dict))
otherattackfile.write('\n')
otherattackfile.close()
# return
except Exception as e:
record_err.logrecord()
def catagory(idsdata):
try:
attack_type = 0
other_alert = open('logs/other_alert.txt', 'a')
httpcatafile = open('logs/http_cat.txt', 'a')
idsdata['catagory'] = ''
alert = idsdata['alert']
# CVE
if 'CVE' in alert:
idsdata['catagory'] = 'cve_attack'
# read file
elif 'read File' in alert:
idsdata['catagory'] = 'read_file'
elif 'aexp2.htr access' in alert:
idsdata['catagory'] = 'read_file'
elif 'EXPLOIT .htr access' in alert:
idsdata['catagory'] = 'read_file'
elif '.htaccess access' in alert:
idsdata['catagory'] = 'read_file'
elif '.ida access' in alert:
idsdata['catagory'] = 'read_file'
elif '.idq access' in alert:
idsdata['catagory'] = 'read_file'
elif 'idq attempt' in alert:
idsdata['catagory'] = 'read_file'
elif 'ida attempt' in alert:
idsdata['catagory'] = 'read_file'
elif '.asa access' in alert:
idsdata['catagory'] = 'read_file'
elif 'viewcode access' in alert:
idsdata['catagory'] = 'read_file'
elif 'printenv access' in alert:
idsdata['catagory'] = 'read_file'
elif 'fpcount access' in alert:
idsdata['catagory'] = 'read_file'
elif 'mod_gzip_status access' in alert:
idsdata['catagory'] = 'read_file'
elif 'executable downloader high likelihood' in alert:
idsdata['catagory'] = 'read_file'
elif '.cnf access' in alert:
idsdata['catagory'] = 'read_file'
elif 'Tomcat directory traversal attempt' in alert:
idsdata['catagory'] = 'read_file'
elif 'phpinfo access' in alert:
idsdata['catagory'] = 'read_file'
elif 'cnf access' in alert:
idsdata['catagory'] = 'read_file'
# xxe
elif 'XXE' in alert:
idsdata['catagory'] = 'xxe_attack'
# iis
elif 'EXPLOIT iisadmpwd' in alert:
idsdata['catagory'] = 'iis_attack'
elif 'iissamples access' in alert:
idsdata['catagory'] = 'iis_attack'
elif 'ASP file access' in alert:
idsdata['catagory'] = 'iis_attack'
# xss
elif 'Cross Site Scripting' in alert:
idsdata['catagory'] = 'xss_attack'
# url scan
elif 'administrator access' in alert:
idsdata['catagory'] = 'url_scan'
# sql inject
elif 'SQL Errors in HTTP 200' in alert:
idsdata['catagory'] = 'sql_inject'
# command execute
elif 'Access to /phppath/php' in alert:
idsdata['catagory'] = 'cmd_execute'
elif 'System Command' in alert:
idsdata['catagory'] = 'cmd_execute'
elif 'PHP config option' in alert:
idsdata['catagory'] = 'cmd_execute'
elif 'PHP tags in HTTP' in alert:
idsdata['catagory'] = 'cmd_execute'
elif 'perl command attempt' in alert:
idsdata['catagory'] = 'cmd_execute'
# scanner
elif 'SCAN Nessus' in alert:
idsdata['catagory'] = 'scanner'
else:
attack_type = 1
#"这里记录的是没有被规则命中的"
other_alert.write(str(idsdata))
other_alert.write('\n')
return
# here need back to the next line
if attack_type == 0:
httpcatafile.write(str(idsdata))
httpcatafile.write('\n')
# deldupl(idsdata)
except Exception as e:
record_err.logrecord()
