def Get_message(self,url,date): db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd,timeout=30) now=timeutil.format("%Y-%m-%d",time.time()) proxy=None count=10 while True: try: #proxy=None myPage=webutil.request(url,timeout=10,proxy=proxy,encoding="gbk") break except Exception as e3: print e3 proxy = proxyutils.choice_proxy(is_debug=False,host="master1",port=8880) if count<=0: raise Exception(u"连续10次失败,放弃") count-=1 time.sleep(1) tree=etree.HTML(myPage) jiben=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[2]/div/ul",num=0,split=u" ")#.//*[@id='container']/div[3]/div[2]/div/ul/li[2] text=xpathutil.get_all_text(tree,".//*[@id='wordDiv']/div/div",num=0,split=u" ") print len(text) if len(text)<=10: text=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]",num=0,split=u" ") else: pass p=functions.remove_all_space_char(text) p=functions.remove_all_space_char(p) emeail=re.compile('[\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+') e=emeail.findall(p) a=[] phone_number=re.compile('^(?:\+86)?(\d{3})\d{8}$|^(?:\+86)?(0\d{2,3})\d{7,8}$') pn=phone_number.findall(p) posdict=dict() for key in self.key_word: found=False for j in key: index=p.find(j) if index>=0: if found: print "error" else: posdict[j]=index found=True for key in posdict: a.append(posdict[key]) a.sort() save_data=dict() for i in range(0,len(a)): if i+1<len(a): text3='' text3=p[int(a[i]):int(a[i+1])].replace(':',':',1).replace(":",":",1).replace(":",":",1) text3=text3.split(':') if len(text3)>1: if len(text3)==2: save_data[text3[0]]=text3[1] else: save_data[text3[0]]=text3[1]+text3[2] elif len(text3)==1: save_data[text3[0]]='None' else: pass else: pass if e: save_data["邮箱"]=e[0] else: save_data["邮箱"]="无" if pn: save_data['电话']=pn[0] else: save_data["电话"]="无" myPage=myPage.encode('utf-8') title=re.findall('<title>(.*?)</title>',myPage) if not title: title=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[1]/h1/a",num=0,split=u" ") name= title else: name=title[0] print name p='' if not jiben: pp=re.findall('<div class="info clearfix"><ol><li>\xe5\x8f\x91\xe5\xb8\x83\xe6\x97\xb6\xe9\x97\xb4:<u>(.*?)</u></li><li>\xe5\xb7\xa5\xe4\xbd\x9c\xe5\x9c\xb0\xe7\x82\xb9:<u>(.*?) </u></li><li>\xe8\x81\x8c\xe4\xbd\x8d\xe7\xb1\xbb\xe5\x9e\x8b:<u>(.*?)</u></li><li>\xe6\x9d\xa5\xe6\xba\x90:<a href="#" onclick="window.open(.*?)">(.*?)</a></li>',myPage) for i in pp: for j in i: if j.find('(')==-1: p=p+j+'\r\n' else: jiben=jiben.replace('\t','') jiben=jiben.split('\r\n') for i in jiben: p=p+i.split(':')[-1].replace('\n','')+'\r\n' print p myPage=myPage.decode('utf-8') keys=url+now save_data["公司名称"]=name save_data["发布时间"]=date save_data["文本1"]=p save_data['文本2']=text save_data["页面链接"]=url save_data["页面源码"]=myPage save_data['dotime']=now save_data['uptime']=time.time() save_data['source']="yingjiesheng" save_data["type"]="1" mongoutil.updatev3(db_yjs,keys,save_data) print("数据入库成功!")
def Clean(self,key): try: db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd,timeout=30) hh=db_yjs.find({'type':'1'}) number=0 for i in hh: if i: p=i.get(u"文本2") id=i.get("_id") p=functions.remove_all_space_char(p) emeail=re.compile('[\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+') #p=p.decode('utf-8','ignore') ' #print p e=emeail.findall(p) a=[] phone_number=re.compile(r'^0\d{2,3}\d{7,8}$|^1[358]\d{9}$|^147\d{8}$') pn=phone_number.findall(p) posdict=dict() #print p[53:102] for key in self.key_word: found=False for j in key: index=p.find(j) if index>=0: if found: print "error" else: posdict[j]=index found=True for key in posdict: a.append(posdict[key]) a.sort() save_data=dict() for i in range(0,len(a)): if i+1<len(a): text3='' text3=p[int(a[i]):int(a[i+1])].replace(':',':',1).replace(":",":",1).replace(":",":",1) text3=text3.split(':') if len(text3)>1: if len(text3)==2: save_data[text3[0]]=text3[1] else: save_data[text3[0]]=text3[1]+text3[2] elif len(text3)==1: save_data[text3[0]]='None' else: pass else: pass if e: save_data["邮箱"]=e[0] else: save_data["邮箱"]="None" if pn: save_data['电话']=pn[0] else: save_data["电话"]="None" save_data['文本3']='None' for i in save_data: print i,save_data[i] number=number+1 mongoutil.updatev3(db_yjs,id,save_data) print "更新成功!%s"%number else: pass logging.error("完毕!") logging.error("%s"%number) except Exception as e2: print e2 logging.error("错误:%s" %e2) logging.error("数量%s" %number) self.Clean(key)
def save(self,company_name,save_data1): """ 通过公司名和日期生成唯一id,并把公司内容存入mongodb数据库。 :param company_name: (unicode) 公司名 :param save_data: (dict) 公司信息 :return: (bool) 是否成功存储 -> true / false """ #清理数据 save_data=dict() for key in save_data1: value=save_data1[key] new_key=remove_all_space_char(key) if len(new_key)>0: save_data[new_key]=value company_name = remove_all_space_char(company_name) fields=[u"成员出资总额",u'名称',u'注册号',u'登记机关',u'类型',u'经营状态',u'登记状态',u'营业场所',u'住所',u'营业期限自',u'营业期限至',u'成立日期',u'核准日期',u'吊销日期',u'注册资本',u"经营期限至",u"经营期限自"] people=[u"名称",u"经营者",u"法定代表",u"法定代表人",u"经营者姓名",u'负责人',u"法人",u"首席代表",u"投资人",u"执行事务合伙人",u"执行事务合伙人(委派代表)",u"股东"] for p in people: if save_data.has_key(p): value=self.parse_people(save_data[p]) save_data[p]=value if len(value)<1: self.logging.error(u"字段内容长度为0,公司名:%s,字段名:%s"%(company_name,p)) for field in fields: if save_data.has_key(field): value=remove_all_space_char(save_data[field]) save_data[field]=value if len(value)<1: self.logging.error(u"字段内容长度为0,公司名:%s,字段名:%s"%(company_name,field)) if len(save_data)<15: raise Exception(u"字段缺失:%s"%company_name) self.logging.info(u"存储数据,公司名:%s"%company_name) now=timeutil.format("%Y-%m-%d",time.time()) id=mongoutil.get_id_key(company_name,now) # prefix="^"+company_name # ret=self.db_qyxx.table.find({"_id":{'$regex':prefix}}).sort("uptime",-1) #处理version,如果未指定则默认为1 if not save_data.has_key("version"): save_data["version"]=3 #处理没有type,如果未指定则默认为chinese if not save_data.has_key("type"): save_data["type"]=self.chinese save_data["company_name"] = company_name #设置公司名字 save_data["do_time"]=now save_data["uptime"]=time.time() save_data["down_type"]=0 #处理键值为None if save_data.has_key(None): del save_data[None] #处理股东信息 gdxx_list=list() if save_data.has_key("gdxx"): gdxx_list=save_data["gdxx"] if not isinstance(gdxx_list,list): gdxx_list=list() save_data["gdxx"]=JSONEncoder().encode(gdxx_list) #处理备案信息 baxx_list=list() if save_data.has_key("baxx"): baxx_list=save_data["baxx"] if not isinstance(baxx_list,list): baxx_list=list() save_data["baxx"]=JSONEncoder().encode(baxx_list) #处理变更信息 bgxx_list=list() if save_data.has_key("bgxx"): bgxx_list=save_data["bgxx"] if not isinstance(bgxx_list,list): bgxx_list=list() save_data["bgxx"]=JSONEncoder().encode(bgxx_list) #处理分支机构 fzjg_list=list() if save_data.has_key("fzjg"): fzjg_list=save_data["fzjg"] if not isinstance(fzjg_list,list): fzjg_list=list() save_data["fzjg"]=JSONEncoder().encode(fzjg_list) #处理行政处罚 xzcf_list=list() if save_data.has_key("xzcf") and isinstance(save_data["xzcf"],list): xzcf_list=save_data["xzcf"] save_data["xzcf"]=JSONEncoder().encode(xzcf_list) # 注册号策略 save_data['keyword'] = self.keyword key_list =save_data.keys() res_list = filter(lambda x: u"注册号" in x , key_list) if not res_list: self.logging.error(u'没有注册号!') if self.is_num(self.keyword) and len(self.keyword) == 15: self.logging.info(u'写入注册号:%s' % self.keyword) save_data[u'注册号'] = self.keyword save_data['_id'] = id save_data['has_company'] = 1 self.db.save(save_data) self.logging.info(u'成功写入%s一条数据:%s' % (config.type1, id)) self.proxy_series_error=0 if self.proxy and self.proxy.split(":")[-1]not in ['42271','42272']: self.logging.info(u"优质非自建代理插入队列尾部,当前非自建代理列表长度为:%s"% self.put_proxy_into_queue_or_set(type='queue')) if not config.debug: self.monitor.add() return True