Exemplo n.º 1
0
 def valid_proxy(self):
     half_proxy = int(self.size("set:proxy:self_built")) / 2
     proxys = self.redis_set_srandmember("set:proxy:self_built", half_proxy)
     for proxy in proxys:
         self.proxy = proxy
         if self.proxy_true():
             return
     from common.proxyutils import choice_proxy
     self.proxy = choice_proxy(is_debug=True,
                               url="",
                               area=u"中国",
                               host=config.proxy_host,
                               port=config.proxy_port)
    def Get_href(self,url):
        RedisQueue=RedisQueue_master1.getredisQueuev2('yingjiesheng_href1')
        f1=open('href.txt','ar+')
        proxy=None
        count=10
        while True:
            try:
                
                #proxy=None
                myPage=webutil.request(url,timeout=10,proxy=proxy).decode('gbk').encode('utf-8')
                break
            except Exception as e3:
                print e3
		proxy = proxyutils.choice_proxy(is_debug=False,host="master1",port=8880)

                if count<=0:
                    raise  Exception(u"连续10次失败,放弃")
                count-=1
                time.sleep(1) 
        try:
            pp=re.findall('<a href="(.*?)" target="_blank"><span style="color:(.*?);">(.*?)</a>(.*?)</td>(.*?)<td class="date">(.*?)</td>',myPage,re.S)
            for i in pp:
                m=i[0]
                p=m.split('href="')[-1]
                if p.find('http://')==-1:
                    href='http://www.yingjiesheng.com'+p
                else:
                    href=p
                if href not in f1.read(): 
                    print href,i[-1]

                    hh=href+'BBD'+i[-1]
                    RedisQueue.put(hh)
                    f1.write(href)
                    f1.write('\r\n')
                    f1.write(i[-1])
                    f1.write('\r\n')
            
            print len(pp)
            f1.close()
        except Exception as e2:
            logging.error("product id:%s" %e2)
            print e2
            pass
    def Get_message(self,url,date):
        db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd,timeout=30)
        now=timeutil.format("%Y-%m-%d",time.time())
        proxy=None
        count=10
        while True:
            try:
               
                #proxy=None
                myPage=webutil.request(url,timeout=10,proxy=proxy,encoding="gbk")
                break
            except Exception as e3:
                print e3
		proxy = proxyutils.choice_proxy(is_debug=False,host="master1",port=8880)
                if count<=0:
                    raise  Exception(u"连续10次失败,放弃")
                count-=1
                time.sleep(1) 
        tree=etree.HTML(myPage)
        
        jiben=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[2]/div/ul",num=0,split=u" ")#.//*[@id='container']/div[3]/div[2]/div/ul/li[2]
        text=xpathutil.get_all_text(tree,".//*[@id='wordDiv']/div/div",num=0,split=u" ")
        print len(text)
        if len(text)<=10:
            text=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]",num=0,split=u" ")
        else:
            pass
	    p=functions.remove_all_space_char(text)
        p=functions.remove_all_space_char(p)
        emeail=re.compile('[\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+')
                    
        e=emeail.findall(p)
        a=[]
        phone_number=re.compile('^(?:\+86)?(\d{3})\d{8}$|^(?:\+86)?(0\d{2,3})\d{7,8}$')
        pn=phone_number.findall(p)
        posdict=dict()
                   
        for  key in self.key_word:
		found=False
                for j in key:
			index=p.find(j)
                        if index>=0:
				if found:
                                    print "error"
                                else:
                                    posdict[j]=index
                                    found=True
        for key in  posdict:
		a.append(posdict[key])
	a.sort()
	save_data=dict()
	for i in range(0,len(a)):
		if i+1<len(a):
			text3=''
                        text3=p[int(a[i]):int(a[i+1])].replace(':',':',1).replace(":",":",1).replace(":",":",1)
                        text3=text3.split(':')

                        if len(text3)>1:
				if len(text3)==2:
                                        save_data[text3[0]]=text3[1]

                                else:
                                        save_data[text3[0]]=text3[1]+text3[2]
                        elif len(text3)==1:
                               	save_data[text3[0]]='None'
                        else:
				pass
		
		else:
			pass

        if e:
            save_data["邮箱"]=e[0]
        else:
            save_data["邮箱"]="无"
        if pn:
               	save_data['电话']=pn[0]
        else:
                save_data["电话"]="无"
    
	
        myPage=myPage.encode('utf-8')
        title=re.findall('<title>(.*?)</title>',myPage)
        if not title:
            title=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[1]/h1/a",num=0,split=u" ")
            name= title
        else:
            name=title[0]
        print name

        p=''
        if not jiben:
            pp=re.findall('<div class="info clearfix"><ol><li>\xe5\x8f\x91\xe5\xb8\x83\xe6\x97\xb6\xe9\x97\xb4:<u>(.*?)</u></li><li>\xe5\xb7\xa5\xe4\xbd\x9c\xe5\x9c\xb0\xe7\x82\xb9:<u>(.*?) </u></li><li>\xe8\x81\x8c\xe4\xbd\x8d\xe7\xb1\xbb\xe5\x9e\x8b:<u>(.*?)</u></li><li>\xe6\x9d\xa5\xe6\xba\x90:<a href="#" onclick="window.open(.*?)">(.*?)</a></li>',myPage)
            for i in pp:
                for j in i:
                    if j.find('(')==-1:
                        p=p+j+'\r\n'
        else:
            jiben=jiben.replace('\t','')
            jiben=jiben.split('\r\n')
            for i in jiben:
                p=p+i.split(':')[-1].replace('\n','')+'\r\n' 
        print p 
        myPage=myPage.decode('utf-8')
        keys=url+now
	save_data["公司名称"]=name
	save_data["发布时间"]=date
	save_data["文本1"]=p
	save_data['文本2']=text
	save_data["页面链接"]=url
	save_data["页面源码"]=myPage
	save_data['dotime']=now
	save_data['uptime']=time.time()
	save_data['source']="yingjiesheng"
	save_data["type"]="1"
        mongoutil.updatev3(db_yjs,keys,save_data)
	print("数据入库成功!")