def __init__(self, key=''):
     '''
     完成初始化工作,version可以任意,但是格式必须为*.*,api_version必须为3.1版本,key为google api key,需在网上申请
     '''
     self.key = key
     self.version = '1.0'
     self.api_version = '3.1'
     if self.key == '':
         raise ValueError("缺少Google API,请于 Google Developers Console中申请 API Key")
     self.url_google = 'https://sb-ssl.google.com/safebrowsing/api/lookup?client=%s&key=%s&appver=%s&pver=%s' % ('python', self.key, self.version, self.api_version)
     mysql = MysqlConnection()
     self.conn = mysql.return_conn()
def main():
    conn = MysqlConnection().return_conn()
    cursor = conn.cursor()
    cursor.execute('select ip from ip_location')
    IPL = IPLocator( )
    
    for ip in cursor.fetchall():
        address = IPL.getIpAddr( IPL.str2ip(str(ip[0])) )   #str2ip使查询ip的格式与数据库中的相同
        print ip[0],address
        
    IPL.getIndexCount()      #输出版本信息和记录总数
    cursor.close()
    conn.close()
def main():
    conn = MysqlConnection().return_conn()
    cursor = conn.cursor()
    cursor.execute('select ip from ip_location')
    IPL = IPLocator()

    for ip in cursor.fetchall():
        address = IPL.getIpAddr(IPL.str2ip(str(
            ip[0])))  #str2ip使查询ip的格式与数据库中的相同
        print ip[0], address

    IPL.getIndexCount()  #输出版本信息和记录总数
    cursor.close()
    conn.close()
 def __init__(self):
     '''
     初始化,key需要在phishtank网站申请,以及数据网址
     '''
     mysql = MysqlConnection()
     self.conn = mysql.return_conn()
     key =  self.phishtank_key_selection() #得到有效phitank开发者api
         
     if str(key) == "False": #没有可用key
         exit(0)
 
     self.key = key
     self.phishsite = "http://data.phishtank.com/data/"
     self.fileformat = '/online-valid.json'
     self.total = 0.0  #数据大小
def domain2ip_batch():
    """批量解析"""
    
    sql = 'select domain from url_detail_info limit 20'
    mysql = MysqlConnection()
    conn = mysql.return_conn()
    cursor = conn.cursor()
    cursor.execute(sql)
    ips = cursor.fetchall()
    
    for ip in ips:
        try:
            result = socket.getaddrinfo(ip[0], 'http')
            print ip[0],result
        except socket.error, err_msg:
            print err_msg #回显异常信息
            continue
Example #6
0
def main():

    sql = 'SELECT url from url_detail_info limit 1000'
    conn = MysqlConnection().return_conn()
    cursor = conn.cursor()
    cursor.execute(sql)
    url_list = cursor.fetchall()
    rowcount = cursor.rowcount   #需要检测的url数量

    count = 0

    while count*THREAD_NUM < rowcount:
        input_urls = url_list[count * THREAD_NUM:(count+1) * THREAD_NUM]
        for url in input_urls:
            t = threading.Thread(target = online,args = (url[0],rowcount,))
            t.start()
        count += 1
        time.sleep(2)  #防止线程过多,造成丢包

    cursor.close()
    conn.close()
def main():

    #获取需要检测urls列表
	sql = 'select id,url from url_detail_info where virustotal_detail is NULL or virustotal_detail = "" limit 1'
	conn = MysqlConnection().return_conn()
	cursor = conn.cursor()
	cursor.execute(sql)
	urls = cursor.fetchall()
	cursor.close()
	conn.close()
    
	if len(urls) == 0:
		print 'There is not url needs to be scaned with virustotal'
		sys.exit(0)

	for url in urls:
        
		url_scan = VirusTotalUrls(url[1])
		content = url_scan.retrieving_url_report()
		if not content:
			print url[1] + " , Please check this url later"
			continue
		url_scan.analysis_report(url[0],content)
class File2Sql:
    
    def __init__(self,file_source = 'ip_block_source/anhui.txt'):
        '''初始化函数'''
        
        self.conn = MysqlConnection().return_conn()  #连接数据库
        self.file_source = file_source                                     #ip段txt文件路径和文件名
        
    def file2sql(self,num = 500):
        '''文件ip段导入到数据库中'''
                             
        sql = 'INSERT INTO ip_block (start_ip,end_ip,region_id,state) VALUES (%s,%s,"1","0")'
        cursor = self.conn.cursor()
        ip_block = file(self.file_source,mode = 'r')           #打开文件
        ip_block_lines = ip_block.readlines()                     #得到所有ip段
        linecount = len(ip_block_lines)                                #得到行数
        for line in range(linecount):
            list_ip = ip_block_lines[line].strip().split('\t')      #分隔为列表
            long_start_ip = ip2long(str(list_ip[0]))
            long_end_ip = ip2long(str(list_ip[1]))
            row_count = long_end_ip - long_start_ip+1
            counts = row_count/num
            for count in range(counts):
                change_ip = long_start_ip + num
                if change_ip<long_end_ip:
                    print long2ip(long_start_ip),long2ip(change_ip)
                    cursor.execute(sql,(long2ip(long_start_ip),long2ip(change_ip)))
                    long_start_ip = change_ip+1
                else:
                    print long2ip(long_start_ip),long2ip(long_end_ip)
                    cursor.execute(sql,(long2ip(long_start_ip),long2ip(change_ip)))
                    break
            self.conn.commit()
                                  
        ip_block.close()
        cursor.close()
        self.conn.close()
    def __init__(self,DHOST='114.114.114.114'):
        '''
        初始化类DomainToIp,连接数据库,DNS服务器ip地址可以更改
        '''
        
        self.DHOST = DHOST                     #DNS 服务器的地址
        self.DPORT = 53                        #默认端口是53
        self.tid = random.randint(0,65535)     #tid为随机数
        self.opcode = Opcode.QUERY             #标准查询

        self.qtype = Type.A                                 #查询类型为A
        self.qclass = Class.IN                              #查询类IN
        self.rd = 1                                                   #期望递归查询
        
        self.mysql = MysqlConnection()        #连接数据库
        self.conn = self.mysql.return_conn()
	def analysis_report(self,url_id,content):
		"""解析检测报告,并将结果返回,否则返回False"""
				
		conn = MysqlConnection().return_conn()
		cursor = conn.cursor()

		response_code = json.loads(content)['response_code']
		if response_code == 0:                 #不同的response_code,代表情况不同
			print self.url + ' ,This site is not present virustotal\'s  database'
		elif response_code == -2:
			print self.url + ' ,the requested item is still queued for analysis'
		elif response_code == 1:
			url = json.loads(content)['url']               #查询的url
			scan_date = json.loads(content)['scan_date']   #扫描时间
			positives = json.loads(content)['positives']   #是恶意网址的扫描引擎数量
			total = json.loads(content)['total']           #总共使用扫描引擎数量
			scans = json.loads(content)['scans']

			result_list=[]
			i = 0
			for key in scans.keys():
				if  scans[key]['detected'] == True:
					result_list.append({str(key):str(scans[key]['result'])})
					i = i + 1

			sql = 'INSERT INTO virustotal_details(id,url,subtime,total,positives' + createfield(i) + ')'
			num = ' "%s"' * i
			li  = num.split(' ')
			sql = sql + ' VALUES ("%s","%s","%s","%s","%s" ' + ','.join(li) + ')'


			value = (str(url_id),str(url),str(scan_date),str(total),str(positives))+tuple(result_list)
			sql = sql % value
			print sql
			
			cursor.execute(sql)
			conn.commit()
			cursor.close()
			conn.close()
#!/usr/bin/python
#encoding:utf-8

import sys
from mysql_connection import MysqlConnection
reload(sys)
sys.setdefaultencoding( "utf-8" )

if len(sys.argv) <3:
    print "wrong format,eg. python check_malicious_url.py  input.txt  output.txt"
    sys.exit(0)

fr_check_url = open(sys.argv[1],'r')
fw_result_url = open(sys.argv[2],'w')

mysql = MysqlConnection()
conn = mysql.return_conn()
cursor = conn.cursor() 
check_url_list = fr_check_url.readlines()
for url in check_url_list:

    sql = 'SELECT * FROM url_detail_info WHERE url LIKE ' +'\"' + str(url.strip())+'%'+'\"' 
    cursor.execute(sql)
    result = cursor.fetchall()
    if not result:
        fw_result_url.write(url.strip() + '\t' + 'No' +'\n')
    else:
        fw_result_url.write(url.strip() + '\t' + 'Yes' +'\n')

print 'Check end'
fr_check_url.close()
 def __init__(self,file_source = 'ip_block_source/anhui.txt'):
     '''初始化函数'''
     
     self.conn = MysqlConnection().return_conn()  #连接数据库
     self.file_source = file_source                                     #ip段txt文件路径和文件名
Example #13
0
    dir_url = c.getinfo(c.EFFECTIVE_URL)  #得到重定向网址
    print checkurl + ' ' + str(http_code) +' ' + dir_url
    
    c.close()
    b.close()

    lock.acquire()  #线程锁

    result_urls.append({'url': checkurl,'http_code': http_code,'dir_url': dir_url})
    URL_COUNT += 1

    if URL_COUNT % RESULTS_NUM == 0 or URL_COUNT == rowcount:   #每隔RESULTS_NUM个url时候,将结果存入到数据库中
        print result_urls

        sql = 'UPDATE url_detail_info SET online = %s WHERE url = %s '
        conn = MysqlConnection().return_conn()
        cursor = conn.cursor()
        for url in result_urls:
            cursor.execute(sql,(url['http_code'],url['url']))

        conn.commit()   #更新
        cursor.close()  #关闭
        conn.close()
    
        del result_urls[:]      #清除已存入的url探测结果
        if URL_COUNT == rowcount:
            print 'end'
          
    lock.release() #解锁

Example #14
0
 def  __init__(self):
     ''' 初始化,连接数据库'''
     
     self.mysql = MysqlConnection()
     self.conn = self.mysql.return_conn()
Example #15
0
class PhishsiteData:
    '''
    phishwebsite类
    '''
    def  __init__(self):
        ''' 初始化,连接数据库'''
        
        self.mysql = MysqlConnection()
        self.conn = self.mysql.return_conn()
        
    def json2mysql(self):
        '''
        把json导入mysql数据库data表中
        '''
        
        filedata = open('data.json')
        jsdata = json.load(filedata)
        cursor = self.conn.cursor()

        cursor.execute('SELECT hash from url_detail_info')
        hash_list = cursor.fetchall()


        sql = "INSERT INTO  url_detail_info (url,domain,domain_type,domain_info,type,target,submission_time,verification_time,online,verified,url_source,hash,virustotal_detail) VALUES( %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        update_num = 0      #统计本次更新url数量
        
        print 'Updating....'


        try:

            for item in jsdata:   #添加最新的url
    

                url_hash = hash(item['url'])
                if not (str(url_hash),) in hash_list: #使用hash值来去重

                    print 'Add url : ' + item.get('url','0')

                    url = item.get('url','0')  #得到url
                    domain = urlparse(item.get('url','0')).netloc #得到domain
                    if not item.get('details'): 
                        ip_address = '0'
                    else:
                        ip_address = item.get('details','0')[0].get('ip_address','0') #得到ip
                    target = item.get('target','0')                      #得到target
                    submission_time = item.get('submission_time','0')    #得到submission_time
                    verification_time = item.get('verification_time','0')#得到verification_time
                    online = item.get('online','0')                      #得到online
                    verified = item.get('verified','0')                  #得到verified
                    phish_id = item.get('phish_id','0')                  #得到phish_id

                    update_num = update_num + cursor.execute(sql,(url,domain,'1',ip_address,'phishing',target,submission_time,verification_time,online,verified,phish_id,url_hash,'1'))
                    cursor.execute('INSERT INTO virustotal_info (url,hash,source) VALUES(%s,%s,%s) ',(url,url_hash,'1'))
                    
                    if update_num % 800 == 0: #每800个result插入到数据库中
                        self.conn.commit()

            self.conn.commit()       #更新

            print 'Success update '+str(update_num)+' url(s)'

            filedata.close()
            cursor.close()
            self.conn.close()

        except MySQLdb.Error,e:  #异常处理
            print "Mysql Error %d: %s" % (e.args[0], e.args[1])
class DomainToIp:
    
    def __init__(self,DHOST='114.114.114.114'):
        '''
        初始化类DomainToIp,连接数据库,DNS服务器ip地址可以更改
        '''
        
        self.DHOST = DHOST                     #DNS 服务器的地址
        self.DPORT = 53                        #默认端口是53
        self.tid = random.randint(0,65535)     #tid为随机数
        self.opcode = Opcode.QUERY             #标准查询

        self.qtype = Type.A                                 #查询类型为A
        self.qclass = Class.IN                              #查询类IN
        self.rd = 1                                                   #期望递归查询
        
        self.mysql = MysqlConnection()        #连接数据库
        self.conn = self.mysql.return_conn()
        
    def send_domain_receive_ip(self):
        '''
        解析domain name对应的ip,并保存到数据库
        '''
      
        sql = 'SELECT domain from url_detail_info limit 20' #where domain_type is NULL OR domain_type = ""'
        cursor = self.conn.cursor()
        cursor.execute(sql)
        domains = cursor.fetchall()    #获取数据库中ip为空的数据
        
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)                               #建立一个UDP套接字(SOCK_DGRAM,代表UDP,AF_INET表示IPv4)
        except socket.error,msg:
            print "无法创建socket.Error code:" +str(msg[0])+',Error message:'+msg[1]    #error
            sys.exit(1)
        source_port = random.randint(1024, 65535)                                                                #随机port
        s.bind(('', source_port))                                                                                                         #绑定,检测所有接口
        
        
        domain_source = []      #发送的domain数量
        domain_result = []      #接收到的domain数量,这两个变量主要用来判断丢包情况
        result = []         #得到的结果
        '''循环发送需要解析的domain name''' 

        count = 0
        rowcount = len(domains)



        while count * UPDATE_RATE < rowcount:  #google每次最多查询500个
            inputs = domains[count * UPDATE_RATE : (count + 1) * UPDATE_RATE]

            for domain in inputs:
                
                domain_source.append(domain[0])
                
                m = Lib.Mpacker()
                m.addHeader(self.tid, 0, self.opcode, 0, 0, self.rd, 0, 0, 0, 1, 0, 0, 0)
                m.addQuestion(domain[0],self.qtype,self.qclass)
                request = m.getbuf()
                try:
                    s.sendto(request,(self.DHOST, self.DPORT))
                    print 'domain: ',domain[0]," send to Dns server:",self.DHOST
                except socket.error,reason:
                    print  reason
                    continue
                
            # result = []         #得到的结果
            
            '''循环接收收到的返回header'''   
            while 1:
                try:
                    r,w,e = select.select([s], [], [],3)
                    if not (r or w or e):
                        break
                    (data,addr) = s.recvfrom(65535)
                    u = Lib.Munpacker(data)
                    r = Lib.DnsResult(u,{})

                    
                    if r.header['status'] == 'NOERROR':
                        #print 'answers',len(r.answers),r.questions[0]['qname']

                        if len(r.answers) != 0:
                        
                            if r.answers[0]['typename'] == 'A':
                                result.append({'domain' : r.questions[0]['qname'],'domain_type': A_FLAG,'domain_info':r.answers[0]['data']})
                                domain_result.append(r.questions[0]['qname'])
                            elif r.answers[0]['typename'] == 'CNAME':
                                result.append({'domain' : r.questions[0]['qname'],'domain_type': CNAME_FLAG,'domain_info':[r.answers[1]['name'],r.answers[1]['data']]})
                                domain_result.append(r.questions[0]['qname'])
                            else:
                                print '没有这种类型,请修改程序'
                        else:
                            result.append({'domain' : r.questions[0]['qname'],'domain_type': ANSWER_EMPTY_FLAG,'domain_info': 'answerempty'}) 
                            domain_result.append(r.questions[0]['qname'])

                    elif r.header['status'] == 'NXDOMAIN':
                        result.append({'domain' : r.questions[0]['qname'],'domain_type': NXDOMAIN_FLAG,'domain_info':[r.authority[0]['name'],r.authority[0]['data'][0]]})
                        domain_result.append(r.questions[0]['qname'])
                    elif r.header['status'] == 'SERVFAIL':
                        result.append({'domain' : r.questions[0]['qname'],'domain_type': SERVFAIL_FLAG,'domain_info': 'servfail'})    #status ='SERVFAIL'情况的判断
                        domain_result.append(r.questions[0]['qname'])
                    else:
                        print 'No this type'
                except socket.error, reason:
                    print reason
                    continue