def __init__(self, key=''):
'''
完成初始化工作，version可以任意，但是格式必须为*.*，api_version必须为3.1版本，key为google api key，需在网上申请
'''
self.key = key
self.version = '1.0'
self.api_version = '3.1'
if self.key == '':
raise ValueError("缺少Google API，请于 Google Developers Console中申请 API Key")
self.url_google = 'https://sb-ssl.google.com/safebrowsing/api/lookup?client=%s&key=%s&appver=%s&pver=%s' % ('python', self.key, self.version, self.api_version)
mysql = MysqlConnection()
self.conn = mysql.return_conn()
def __init__(self):
'''
初始化,key需要在phishtank网站申请，以及数据网址
'''
mysql = MysqlConnection()
self.conn = mysql.return_conn()
key = self.phishtank_key_selection() #得到有效phitank开发者api
if str(key) == "False": #没有可用key
exit(0)
self.key = key
self.phishsite = "http://data.phishtank.com/data/"
self.fileformat = '/online-valid.json'
self.total = 0.0 #数据大小
def domain2ip_batch():
"""批量解析"""
sql = 'select domain from url_detail_info limit 20'
mysql = MysqlConnection()
conn = mysql.return_conn()
cursor = conn.cursor()
cursor.execute(sql)
ips = cursor.fetchall()
for ip in ips:
try:
result = socket.getaddrinfo(ip[0], 'http')
print ip[0],result
except socket.error, err_msg:
print err_msg #回显异常信息
continue
#encoding:utf-8
import sys
from mysql_connection import MysqlConnection
reload(sys)
sys.setdefaultencoding( "utf-8" )
if len(sys.argv) <3:
print "wrong format,eg. python check_malicious_url.py input.txt output.txt"
sys.exit(0)
fr_check_url = open(sys.argv[1],'r')
fw_result_url = open(sys.argv[2],'w')
mysql = MysqlConnection()
conn = mysql.return_conn()
cursor = conn.cursor()
check_url_list = fr_check_url.readlines()
for url in check_url_list:
sql = 'SELECT * FROM url_detail_info WHERE url LIKE ' +'\"' + str(url.strip())+'%'+'\"'
cursor.execute(sql)
result = cursor.fetchall()
if not result:
fw_result_url.write(url.strip() + '\t' + 'No' +'\n')
else:
fw_result_url.write(url.strip() + '\t' + 'Yes' +'\n')
print 'Check end'
fr_check_url.close()
fw_result_url.close()
class PhishsiteData:
'''
phishwebsite类
'''
def __init__(self):
''' 初始化，连接数据库'''
self.mysql = MysqlConnection()
self.conn = self.mysql.return_conn()
def json2mysql(self):
'''
把json导入mysql数据库data表中
'''
filedata = open('data.json')
jsdata = json.load(filedata)
cursor = self.conn.cursor()
cursor.execute('SELECT hash from url_detail_info')
hash_list = cursor.fetchall()
sql = "INSERT INTO url_detail_info (url,domain,domain_type,domain_info,type,target,submission_time,verification_time,online,verified,url_source,hash,virustotal_detail) VALUES( %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
update_num = 0 #统计本次更新url数量
print 'Updating....'
try:
for item in jsdata: #添加最新的url
url_hash = hash(item['url'])
if not (str(url_hash),) in hash_list: #使用hash值来去重
print 'Add url : ' + item.get('url','0')
url = item.get('url','0') #得到url
domain = urlparse(item.get('url','0')).netloc #得到domain
if not item.get('details'):
ip_address = '0'
else:
ip_address = item.get('details','0')[0].get('ip_address','0') #得到ip
target = item.get('target','0') #得到target
submission_time = item.get('submission_time','0') #得到submission_time
verification_time = item.get('verification_time','0')#得到verification_time
online = item.get('online','0') #得到online
verified = item.get('verified','0') #得到verified
phish_id = item.get('phish_id','0') #得到phish_id
update_num = update_num + cursor.execute(sql,(url,domain,'1',ip_address,'phishing',target,submission_time,verification_time,online,verified,phish_id,url_hash,'1'))
cursor.execute('INSERT INTO virustotal_info (url,hash,source) VALUES(%s,%s,%s) ',(url,url_hash,'1'))
if update_num % 800 == 0: #每800个result插入到数据库中
self.conn.commit()
self.conn.commit() #更新
print 'Success update '+str(update_num)+' url(s)'
filedata.close()
cursor.close()
self.conn.close()
except MySQLdb.Error,e: #异常处理
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
class DomainToIp:
def __init__(self,DHOST='114.114.114.114'):
'''
初始化类DomainToIp，连接数据库，DNS服务器ip地址可以更改
'''
self.DHOST = DHOST #DNS 服务器的地址
self.DPORT = 53 #默认端口是53
self.tid = random.randint(0,65535) #tid为随机数
self.opcode = Opcode.QUERY #标准查询
self.qtype = Type.A #查询类型为A
self.qclass = Class.IN #查询类IN
self.rd = 1 #期望递归查询
self.mysql = MysqlConnection() #连接数据库
self.conn = self.mysql.return_conn()
def send_domain_receive_ip(self):
'''
解析domain name对应的ip，并保存到数据库
'''
sql = 'SELECT domain from url_detail_info limit 20' #where domain_type is NULL OR domain_type = ""'
cursor = self.conn.cursor()
cursor.execute(sql)
domains = cursor.fetchall() #获取数据库中ip为空的数据
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) #建立一个UDP套接字（SOCK_DGRAM，代表UDP，AF_INET表示IPv4）
except socket.error,msg:
print "无法创建socket.Error code:" +str(msg[0])+',Error message:'+msg[1] #error
sys.exit(1)
source_port = random.randint(1024, 65535) #随机port
s.bind(('', source_port)) #绑定，检测所有接口
domain_source = [] #发送的domain数量
domain_result = [] #接收到的domain数量，这两个变量主要用来判断丢包情况
result = [] #得到的结果
'''循环发送需要解析的domain name'''
count = 0
rowcount = len(domains)
while count * UPDATE_RATE < rowcount: #google每次最多查询500个
inputs = domains[count * UPDATE_RATE : (count + 1) * UPDATE_RATE]
for domain in inputs:
domain_source.append(domain[0])
m = Lib.Mpacker()
m.addHeader(self.tid, 0, self.opcode, 0, 0, self.rd, 0, 0, 0, 1, 0, 0, 0)
m.addQuestion(domain[0],self.qtype,self.qclass)
request = m.getbuf()
try:
s.sendto(request,(self.DHOST, self.DPORT))
print 'domain: ',domain[0]," send to Dns server:",self.DHOST
except socket.error,reason:
print reason
continue
# result = [] #得到的结果
'''循环接收收到的返回header'''
while 1:
try:
r,w,e = select.select([s], [], [],3)
if not (r or w or e):
break
(data,addr) = s.recvfrom(65535)
u = Lib.Munpacker(data)
r = Lib.DnsResult(u,{})
if r.header['status'] == 'NOERROR':
#print 'answers',len(r.answers),r.questions[0]['qname']
if len(r.answers) != 0:
if r.answers[0]['typename'] == 'A':
result.append({'domain' : r.questions[0]['qname'],'domain_type': A_FLAG,'domain_info':r.answers[0]['data']})
domain_result.append(r.questions[0]['qname'])
elif r.answers[0]['typename'] == 'CNAME':
result.append({'domain' : r.questions[0]['qname'],'domain_type': CNAME_FLAG,'domain_info':[r.answers[1]['name'],r.answers[1]['data']]})
domain_result.append(r.questions[0]['qname'])
else:
print '没有这种类型，请修改程序'
else:
result.append({'domain' : r.questions[0]['qname'],'domain_type': ANSWER_EMPTY_FLAG,'domain_info': 'answerempty'})
domain_result.append(r.questions[0]['qname'])
elif r.header['status'] == 'NXDOMAIN':
result.append({'domain' : r.questions[0]['qname'],'domain_type': NXDOMAIN_FLAG,'domain_info':[r.authority[0]['name'],r.authority[0]['data'][0]]})
domain_result.append(r.questions[0]['qname'])
elif r.header['status'] == 'SERVFAIL':
result.append({'domain' : r.questions[0]['qname'],'domain_type': SERVFAIL_FLAG,'domain_info': 'servfail'}) #status ='SERVFAIL'情况的判断
domain_result.append(r.questions[0]['qname'])
else:
print 'No this type'
except socket.error, reason:
print reason
continue