class IP(Resource):
def __init__(self):
self.mh = MongoHelper()
self.mh.init_db()
@marshal_with(
resource_field, )
def get(self):
print('get')
parser = reqparse.RequestParser()
parser.add_argument('count', type=str)
parser.add_argument('page_index', type=int)
parser.add_argument('page_size', type=int)
args = parser.parse_args()
return self.mh.select(
args['count'] if args['count'] is not None else 5,
page_index=args['page_index'] if args['page_index'] else 1,
page_size=args['page_size'] if args['page_size'] else 5)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('id', type=str)
args = parser.parse_args()
return self.mh.downscore(
args if args['id'] is not None else abort(500))
def getIp():
opener = urllib2.build_opener()
mongohelper = MongoHelper('proxy', 'proxys')
dicts = mongohelper.findAnyOne(True)
#response= opener.open("http://127.0.0.1:8080", timeout=1000)
#dicts=(json.loads(response.read().decode()))
#i=random.randint(0,len(dicts))
ip = str(dicts['ip']) + ':' + str(dicts['port'])
print(ip)
return ip
def get_db():
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
elif DB_CONFIG['DB_CONNECT_TYPE'] == 'redis':
from db.RedisHelper import RedisHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
return sqlhelper
def get_sqlhelper(proxy_type):
try:
#专用代理存储MongoDB
if DB_CONFIG[proxy_type] == 'DEDICATED':
from db.MongoHelper import MongoHelper as SqlHelper
else:
#免费和专用则存储在redis
from db.RedisHelper import RedisHelper as SqlHelper
#from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
#数据库初始化
sqlhelper.init_db()
return sqlhelper
except Exception as e:
raise Con_DB_Fail
def store_data_to_mongo(self):
mongohelper=MongoHelper("zhihu_db", "zhihu_coll")
mongohelper.select_colletion("zhihu_coll")
itemdict={}
itemdict['user_name']= self.user_name
itemdict['user_be_agreed']= self.user_be_agreed
itemdict['user_be_thanked']=self.user_be_thanked
itemdict['user_followees']=self.user_followees
itemdict['user_followers']=self.user_followers
itemdict['user_education_school']=self.user_education_school
itemdict['user_education_subject']=self.user_education_subject
itemdict['user_employment']=self.user_employment
itemdict['user_employment_extra']=self.user_employment_extra
itemdict['user_location']=self.user_location
itemdict['user_gender']=self.user_gender
itemdict['user_info']=self.user_info
itemdict['user_intro']=self.user_intro
itemdict['user_url']=self.url
mongohelper.insert(itemdict)
print "saved: %s \n" %self.user_name
# coding:utf-8
import sys
from config import DB_CONFIG
from util.exception import Con_DB_Fail
try:
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
except Exception as e:
raise Con_DB_Fail
def store_data(queue2, db_proxy_num):
'''
读取队列中的数据，写入数据库中
:param queue2:
:return:
'''
successNum = 0
failNum = 0
while True:
try:
proxy = queue2.get(timeout=300)
if proxy:
sqlhelper.insert(proxy)
pass
# print('预测值: %s, 目标值: %s' % (prediction, y_test[i]))
# print('Predicted: %s, Target: %s' % (prediction, y_test[i]))
print('置信度: %.2f' % models[j].score(X_test, y_test))
print('R-squared: %.2f' % models[j].score(X_test, y_test))
pass
def reSub(instr):
import re
instr = re.sub('[^(\d+(\.\d+)?)]', '', instr)
return instr
if __name__=="__main__":
mongoDB = MongoHelper("autohome", "config")
dataMod1 = [u"厂商指导价", u"排量(mL)", u"最大扭矩转速(rpm)", \
u"发动机", u"最大功率(kW)", u"变速箱", u"最大扭矩(N・m)",\
u"最大马力(Ps)", u"排量(L)"]
dataMod4 = [u"厂商指导价", u"气缸数(个)", u"排量(mL)", \
u"最大扭矩转速(rpm)", \
u"发动机", u"最大功率(kW)", u"变速箱", u"最大扭矩(N・m)",\
u"挡位个数", u"最大马力(Ps)", u"排量(L)", u"每缸气门(个)"]
dataMod3 = [u"厂商指导价", u"气缸数(个)", u"排量(mL)", \
u"最大扭矩转速(rpm)", \
u"发动机", u"最大功率(kW)", u"变速箱", u"最大扭矩(N・m)",\
u"挡位个数", u"最大马力(Ps)", u"排量(L)", u"每缸气门(个)"]
dataMod2 = [u"厂商指导价", u"气缸数(个)", u"排量(mL)", \
u"最大扭矩转速(rpm)", \
u"发动机", u"最大功率(kW)", u"座位数(个)", u"变速箱", u"车门数(个)", u"最大扭矩(N・m)",\
u"挡位个数", u"最大马力(Ps)", u"排量(L)", u"每缸气门(个)"]
# coding:utf-8
import sys
from config import DB_CONFIG
from util.exception import Con_DB_Fail
try:
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
elif DB_CONFIG['DB_CONNECT_TYPE'] == 'redis':
from db.RedisHelper import RedisHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
except Exception as e:
raise Con_DB_Fail
def store_data(queue2, db_proxy_num):
'''
读取队列中的数据，写入数据库中
:param queue2:
:return:
'''
successNum = 0
failNum = 0
while True:
try:
proxy = queue2.get(timeout=300)
if proxy:
pipe = myredis.pipeline()
for i in range(0, len(HEAD_BASE_PHONE)):
for j in range(0000, 9999):
phoneNum = obtainPhoneNum(HEAD_BASE_PHONE[i], j)
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
pipe = myredis.pipeline()
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
if __name__ == "__main__":
import redis
myredis = redis.Redis(db=0, host='127.0.0.1', port=6379)
mongo = MongoHelper()
mongo.init_db()
while (True):
phoneNum = myredis.rpop('phoneNum')
print(phoneNum)
try:
result = getProvider(phoneNum)
print(result)
except IndexError, e:
print(IndexError, e)
pipe = myredis.pipeline()
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
continue
except urllib2.URLError, e:
print(urllib2.URLError, e)
def __init__(self):
self.mh = MongoHelper()
self.mh.init_db()