def store_data_to_mongo(self):
mongohelper=MongoHelper("zhihu_db", "zhihu_coll")
mongohelper.select_colletion("zhihu_coll")
itemdict={}
itemdict['user_name']= self.user_name
itemdict['user_be_agreed']= self.user_be_agreed
itemdict['user_be_thanked']=self.user_be_thanked
itemdict['user_followees']=self.user_followees
itemdict['user_followers']=self.user_followers
itemdict['user_education_school']=self.user_education_school
itemdict['user_education_subject']=self.user_education_subject
itemdict['user_employment']=self.user_employment
itemdict['user_employment_extra']=self.user_employment_extra
itemdict['user_location']=self.user_location
itemdict['user_gender']=self.user_gender
itemdict['user_info']=self.user_info
itemdict['user_intro']=self.user_intro
itemdict['user_url']=self.url
mongohelper.insert(itemdict)
print "saved: %s \n" %self.user_name
mongo.init_db()
while (True):
phoneNum = myredis.rpop('phoneNum')
print(phoneNum)
try:
result = getProvider(phoneNum)
print(result)
except IndexError, e:
print(IndexError, e)
pipe = myredis.pipeline()
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
continue
except urllib2.URLError, e:
print(urllib2.URLError, e)
continue
except socket.timeout, e:
print(IndexError, e)
continue
except:
print("connection close by peer")
continue
print("%s is finished" % phoneNum)
try:
mongo.insert(washData(phoneNum, result))
time.sleep(3)
except IndexError, e:
print(IndexError)
print(e)
#writeResult(result)