def __init__(self, key=''):
     '''
     完成初始化工作,version可以任意,但是格式必须为*.*,api_version必须为3.1版本,key为google api key,需在网上申请
     '''
     self.key = key
     self.version = '1.0'
     self.api_version = '3.1'
     if self.key == '':
         raise ValueError("缺少Google API,请于 Google Developers Console中申请 API Key")
     self.url_google = 'https://sb-ssl.google.com/safebrowsing/api/lookup?client=%s&key=%s&appver=%s&pver=%s' % ('python', self.key, self.version, self.api_version)
     mysql = MysqlConnection()
     self.conn = mysql.return_conn()
 def __init__(self):
     '''
     初始化,key需要在phishtank网站申请,以及数据网址
     '''
     mysql = MysqlConnection()
     self.conn = mysql.return_conn()
     key =  self.phishtank_key_selection() #得到有效phitank开发者api
         
     if str(key) == "False": #没有可用key
         exit(0)
 
     self.key = key
     self.phishsite = "http://data.phishtank.com/data/"
     self.fileformat = '/online-valid.json'
     self.total = 0.0  #数据大小
def domain2ip_batch():
    """批量解析"""
    
    sql = 'select domain from url_detail_info limit 20'
    mysql = MysqlConnection()
    conn = mysql.return_conn()
    cursor = conn.cursor()
    cursor.execute(sql)
    ips = cursor.fetchall()
    
    for ip in ips:
        try:
            result = socket.getaddrinfo(ip[0], 'http')
            print ip[0],result
        except socket.error, err_msg:
            print err_msg #回显异常信息
            continue
#encoding:utf-8

import sys
from mysql_connection import MysqlConnection
reload(sys)
sys.setdefaultencoding( "utf-8" )

if len(sys.argv) <3:
    print "wrong format,eg. python check_malicious_url.py  input.txt  output.txt"
    sys.exit(0)

fr_check_url = open(sys.argv[1],'r')
fw_result_url = open(sys.argv[2],'w')

mysql = MysqlConnection()
conn = mysql.return_conn()
cursor = conn.cursor() 
check_url_list = fr_check_url.readlines()
for url in check_url_list:

    sql = 'SELECT * FROM url_detail_info WHERE url LIKE ' +'\"' + str(url.strip())+'%'+'\"' 
    cursor.execute(sql)
    result = cursor.fetchall()
    if not result:
        fw_result_url.write(url.strip() + '\t' + 'No' +'\n')
    else:
        fw_result_url.write(url.strip() + '\t' + 'Yes' +'\n')

print 'Check end'
fr_check_url.close()
fw_result_url.close()
예제 #5
0
class PhishsiteData:
    '''
    phishwebsite类
    '''
    def  __init__(self):
        ''' 初始化,连接数据库'''
        
        self.mysql = MysqlConnection()
        self.conn = self.mysql.return_conn()
        
    def json2mysql(self):
        '''
        把json导入mysql数据库data表中
        '''
        
        filedata = open('data.json')
        jsdata = json.load(filedata)
        cursor = self.conn.cursor()

        cursor.execute('SELECT hash from url_detail_info')
        hash_list = cursor.fetchall()


        sql = "INSERT INTO  url_detail_info (url,domain,domain_type,domain_info,type,target,submission_time,verification_time,online,verified,url_source,hash,virustotal_detail) VALUES( %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        update_num = 0      #统计本次更新url数量
        
        print 'Updating....'


        try:

            for item in jsdata:   #添加最新的url
    

                url_hash = hash(item['url'])
                if not (str(url_hash),) in hash_list: #使用hash值来去重

                    print 'Add url : ' + item.get('url','0')

                    url = item.get('url','0')  #得到url
                    domain = urlparse(item.get('url','0')).netloc #得到domain
                    if not item.get('details'): 
                        ip_address = '0'
                    else:
                        ip_address = item.get('details','0')[0].get('ip_address','0') #得到ip
                    target = item.get('target','0')                      #得到target
                    submission_time = item.get('submission_time','0')    #得到submission_time
                    verification_time = item.get('verification_time','0')#得到verification_time
                    online = item.get('online','0')                      #得到online
                    verified = item.get('verified','0')                  #得到verified
                    phish_id = item.get('phish_id','0')                  #得到phish_id

                    update_num = update_num + cursor.execute(sql,(url,domain,'1',ip_address,'phishing',target,submission_time,verification_time,online,verified,phish_id,url_hash,'1'))
                    cursor.execute('INSERT INTO virustotal_info (url,hash,source) VALUES(%s,%s,%s) ',(url,url_hash,'1'))
                    
                    if update_num % 800 == 0: #每800个result插入到数据库中
                        self.conn.commit()

            self.conn.commit()       #更新

            print 'Success update '+str(update_num)+' url(s)'

            filedata.close()
            cursor.close()
            self.conn.close()

        except MySQLdb.Error,e:  #异常处理
            print "Mysql Error %d: %s" % (e.args[0], e.args[1])
예제 #6
0
class DomainToIp:
    
    def __init__(self,DHOST='114.114.114.114'):
        '''
        初始化类DomainToIp,连接数据库,DNS服务器ip地址可以更改
        '''
        
        self.DHOST = DHOST                     #DNS 服务器的地址
        self.DPORT = 53                        #默认端口是53
        self.tid = random.randint(0,65535)     #tid为随机数
        self.opcode = Opcode.QUERY             #标准查询

        self.qtype = Type.A                                 #查询类型为A
        self.qclass = Class.IN                              #查询类IN
        self.rd = 1                                                   #期望递归查询
        
        self.mysql = MysqlConnection()        #连接数据库
        self.conn = self.mysql.return_conn()
        
    def send_domain_receive_ip(self):
        '''
        解析domain name对应的ip,并保存到数据库
        '''
      
        sql = 'SELECT domain from url_detail_info limit 20' #where domain_type is NULL OR domain_type = ""'
        cursor = self.conn.cursor()
        cursor.execute(sql)
        domains = cursor.fetchall()    #获取数据库中ip为空的数据
        
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)                               #建立一个UDP套接字(SOCK_DGRAM,代表UDP,AF_INET表示IPv4)
        except socket.error,msg:
            print "无法创建socket.Error code:" +str(msg[0])+',Error message:'+msg[1]    #error
            sys.exit(1)
        source_port = random.randint(1024, 65535)                                                                #随机port
        s.bind(('', source_port))                                                                                                         #绑定,检测所有接口
        
        
        domain_source = []      #发送的domain数量
        domain_result = []      #接收到的domain数量,这两个变量主要用来判断丢包情况
        result = []         #得到的结果
        '''循环发送需要解析的domain name''' 

        count = 0
        rowcount = len(domains)



        while count * UPDATE_RATE < rowcount:  #google每次最多查询500个
            inputs = domains[count * UPDATE_RATE : (count + 1) * UPDATE_RATE]

            for domain in inputs:
                
                domain_source.append(domain[0])
                
                m = Lib.Mpacker()
                m.addHeader(self.tid, 0, self.opcode, 0, 0, self.rd, 0, 0, 0, 1, 0, 0, 0)
                m.addQuestion(domain[0],self.qtype,self.qclass)
                request = m.getbuf()
                try:
                    s.sendto(request,(self.DHOST, self.DPORT))
                    print 'domain: ',domain[0]," send to Dns server:",self.DHOST
                except socket.error,reason:
                    print  reason
                    continue
                
            # result = []         #得到的结果
            
            '''循环接收收到的返回header'''   
            while 1:
                try:
                    r,w,e = select.select([s], [], [],3)
                    if not (r or w or e):
                        break
                    (data,addr) = s.recvfrom(65535)
                    u = Lib.Munpacker(data)
                    r = Lib.DnsResult(u,{})

                    
                    if r.header['status'] == 'NOERROR':
                        #print 'answers',len(r.answers),r.questions[0]['qname']

                        if len(r.answers) != 0:
                        
                            if r.answers[0]['typename'] == 'A':
                                result.append({'domain' : r.questions[0]['qname'],'domain_type': A_FLAG,'domain_info':r.answers[0]['data']})
                                domain_result.append(r.questions[0]['qname'])
                            elif r.answers[0]['typename'] == 'CNAME':
                                result.append({'domain' : r.questions[0]['qname'],'domain_type': CNAME_FLAG,'domain_info':[r.answers[1]['name'],r.answers[1]['data']]})
                                domain_result.append(r.questions[0]['qname'])
                            else:
                                print '没有这种类型,请修改程序'
                        else:
                            result.append({'domain' : r.questions[0]['qname'],'domain_type': ANSWER_EMPTY_FLAG,'domain_info': 'answerempty'}) 
                            domain_result.append(r.questions[0]['qname'])

                    elif r.header['status'] == 'NXDOMAIN':
                        result.append({'domain' : r.questions[0]['qname'],'domain_type': NXDOMAIN_FLAG,'domain_info':[r.authority[0]['name'],r.authority[0]['data'][0]]})
                        domain_result.append(r.questions[0]['qname'])
                    elif r.header['status'] == 'SERVFAIL':
                        result.append({'domain' : r.questions[0]['qname'],'domain_type': SERVFAIL_FLAG,'domain_info': 'servfail'})    #status ='SERVFAIL'情况的判断
                        domain_result.append(r.questions[0]['qname'])
                    else:
                        print 'No this type'
                except socket.error, reason:
                    print reason
                    continue