def insert_to_mongo():
mongodb = mongoutil.getmondbv2("localhost", 27017, "bigdata_higgs",
"qyxx_shangshihangyefenxi")
reader = csv.DictReader(file(u'c:/企业分类数据.csv', 'r'))
for d in reader:
del d[""]
id = d['股票代码']
d['股票代码'] = id.split(".")[0]
d["交易所编号"] = id.split(".")[1]
d["uptime"] = time.time()
mongoutil.updatev3(mongodb, id, d)
def save_data(self, url, data_dic):
if data_dic == None and len(data_dic) == 0 and not isinstance(data_dic, dict):
return
try:
now = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime())
if '_id' not in data_dic:
id = mongoutil.get_id_key(url, now)
else:
id = data_dic['_id']
if None in data_dic:
del data_dic[None]
if 'do_time' not in data_dic:
data_dic['do_time'] = now
if 'uptime' not in data_dic:
data_dic['uptime'] = time.time()
mongoutil.updatev3(self.mongo, id, data_dic)
except Exception as e:
print u'保存数据异常%s' % e
raise Exception(u'保存数据异常')
def Get_message(self,url):
db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd)
now=timeutil.format("%Y-%m-%d",time.time())
proxy=None
count=10
while True:
try:
#proxy = proxyutils.choice_proxy(is_debug=False,host="master1",port=8880)
#proxy=None
myPage=webutil.request(url,timeout=10,proxy=proxy,encoding="gbk")
break
except Exception as e3:
print e3
if count<=0:
raise Exception(u"连续10次失败,放弃")
count-=1
time.sleep(1)
tree=etree.HTML(myPage)
title=xpathutil.get_all_text(tree,".//*[@id='mainNav']/div[2]/table/caption/h1",num=0,split=u" ")
#address=xpathutil.get_all_text(tree,".//*[@id='mainNav']/div[2]/table/tbody/tr[3]/td",num=0,split=u" ")
#pp=re.findall('<td>汉阳郭茨口香格里都3楼腾飞人才市场</td>')
print title
myPage=myPage.encode('utf-8')
address1=re.findall('<th width="90">(.*?)</th>(.*?)<td>(.*?)</td>',myPage,re.S)
j=0
for i in address1:
if j==0:
city1=re.findall('">(.*?)</a>',i[-1])
city=city1[0]
elif j==1:
date=i[-1]
elif j==2:
address=i[-1]
j=j+1
print len(address)
print city
print date
print address
key=url+now
mongoutil.updatev3(db_yjs,key,{"标题":title,"城市":city,"招聘会时间":date,'招聘会地点':address,"页面链接":url,"dotime":now,"uptime":time.time(),"source":"yingjiesheng","type":"2"})
def Clean(self,key):
try:
db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd,timeout=30)
hh=db_yjs.find({'type':'1'})
number=0
for i in hh:
if i:
p=i.get(u"文本2")
id=i.get("_id")
p=functions.remove_all_space_char(p)
emeail=re.compile('[\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+')
#p=p.decode('utf-8','ignore') '
#print p
e=emeail.findall(p)
a=[]
phone_number=re.compile(r'^0\d{2,3}\d{7,8}$|^1[358]\d{9}$|^147\d{8}$')
pn=phone_number.findall(p)
posdict=dict()
#print p[53:102]
for key in self.key_word:
found=False
for j in key:
index=p.find(j)
if index>=0:
if found:
print "error"
else:
posdict[j]=index
found=True
for key in posdict:
a.append(posdict[key])
a.sort()
save_data=dict()
for i in range(0,len(a)):
if i+1<len(a):
text3=''
text3=p[int(a[i]):int(a[i+1])].replace('：',':',1).replace("：",":",1).replace("：",":",1)
text3=text3.split(':')
if len(text3)>1:
if len(text3)==2:
save_data[text3[0]]=text3[1]
else:
save_data[text3[0]]=text3[1]+text3[2]
elif len(text3)==1:
save_data[text3[0]]='None'
else:
pass
else:
pass
if e:
save_data["邮箱"]=e[0]
else:
save_data["邮箱"]="None"
if pn:
save_data['电话']=pn[0]
else:
save_data["电话"]="None"
save_data['文本3']='None'
for i in save_data:
print i,save_data[i]
number=number+1
mongoutil.updatev3(db_yjs,id,save_data)
print "更新成功！%s"%number
else:
pass
logging.error("完毕！")
logging.error("%s"%number)
except Exception as e2:
print e2
logging.error("错误:%s" %e2)
logging.error("数量%s" %number)
self.Clean(key)
def Get_message(self,url,date):
db_yjs=mongoutil.getmondbv2(db.mongo_host,db.mongo_port,db.yjs_db_name,db.yjs_table_name,username=db.mongo_user,password=db.mongo_pwd,timeout=30)
now=timeutil.format("%Y-%m-%d",time.time())
proxy=None
count=10
while True:
try:
#proxy=None
myPage=webutil.request(url,timeout=10,proxy=proxy,encoding="gbk")
break
except Exception as e3:
print e3
proxy = proxyutils.choice_proxy(is_debug=False,host="master1",port=8880)
if count<=0:
raise Exception(u"连续10次失败,放弃")
count-=1
time.sleep(1)
tree=etree.HTML(myPage)
jiben=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[2]/div/ul",num=0,split=u" ")#.//*[@id='container']/div[3]/div[2]/div/ul/li[2]
text=xpathutil.get_all_text(tree,".//*[@id='wordDiv']/div/div",num=0,split=u" ")
print len(text)
if len(text)<=10:
text=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]",num=0,split=u" ")
else:
pass
p=functions.remove_all_space_char(text)
p=functions.remove_all_space_char(p)
emeail=re.compile('[\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+')
e=emeail.findall(p)
a=[]
phone_number=re.compile('^(?:\+86)?(\d{3})\d{8}$|^(?:\+86)?(0\d{2,3})\d{7,8}$')
pn=phone_number.findall(p)
posdict=dict()
for key in self.key_word:
found=False
for j in key:
index=p.find(j)
if index>=0:
if found:
print "error"
else:
posdict[j]=index
found=True
for key in posdict:
a.append(posdict[key])
a.sort()
save_data=dict()
for i in range(0,len(a)):
if i+1<len(a):
text3=''
text3=p[int(a[i]):int(a[i+1])].replace('：',':',1).replace("：",":",1).replace("：",":",1)
text3=text3.split(':')
if len(text3)>1:
if len(text3)==2:
save_data[text3[0]]=text3[1]
else:
save_data[text3[0]]=text3[1]+text3[2]
elif len(text3)==1:
save_data[text3[0]]='None'
else:
pass
else:
pass
if e:
save_data["邮箱"]=e[0]
else:
save_data["邮箱"]="无"
if pn:
save_data['电话']=pn[0]
else:
save_data["电话"]="无"
myPage=myPage.encode('utf-8')
title=re.findall('<title>(.*?)</title>',myPage)
if not title:
title=xpathutil.get_all_text(tree,".//*[@id='container']/div[3]/div[1]/h1/a",num=0,split=u" ")
name= title
else:
name=title[0]
print name
p=''
if not jiben:
pp=re.findall('<div class="info clearfix"><ol><li>\xe5\x8f\x91\xe5\xb8\x83\xe6\x97\xb6\xe9\x97\xb4：<u>(.*?)</u></li><li>\xe5\xb7\xa5\xe4\xbd\x9c\xe5\x9c\xb0\xe7\x82\xb9：<u>(.*?) </u></li><li>\xe8\x81\x8c\xe4\xbd\x8d\xe7\xb1\xbb\xe5\x9e\x8b：<u>(.*?)</u></li><li>\xe6\x9d\xa5\xe6\xba\x90：<a href="#" onclick="window.open(.*?)">(.*?)</a></li>',myPage)
for i in pp:
for j in i:
if j.find('(')==-1:
p=p+j+'\r\n'
else:
jiben=jiben.replace('\t','')
jiben=jiben.split('\r\n')
for i in jiben:
p=p+i.split('：')[-1].replace('\n','')+'\r\n'
print p
myPage=myPage.decode('utf-8')
keys=url+now
save_data["公司名称"]=name
save_data["发布时间"]=date
save_data["文本1"]=p
save_data['文本2']=text
save_data["页面链接"]=url
save_data["页面源码"]=myPage
save_data['dotime']=now
save_data['uptime']=time.time()
save_data['source']="yingjiesheng"
save_data["type"]="1"
mongoutil.updatev3(db_yjs,keys,save_data)
print("数据入库成功！")
