class IP(Resource):
def __init__(self):
self.mh = MongoHelper()
self.mh.init_db()
@marshal_with(
resource_field, )
def get(self):
print('get')
parser = reqparse.RequestParser()
parser.add_argument('count', type=str)
parser.add_argument('page_index', type=int)
parser.add_argument('page_size', type=int)
args = parser.parse_args()
return self.mh.select(
args['count'] if args['count'] is not None else 5,
page_index=args['page_index'] if args['page_index'] else 1,
page_size=args['page_size'] if args['page_size'] else 5)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('id', type=str)
args = parser.parse_args()
return self.mh.downscore(
args if args['id'] is not None else abort(500))
def get_db():
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
elif DB_CONFIG['DB_CONNECT_TYPE'] == 'redis':
from db.RedisHelper import RedisHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
return sqlhelper
def get_sqlhelper(proxy_type):
try:
#专用代理存储MongoDB
if DB_CONFIG[proxy_type] == 'DEDICATED':
from db.MongoHelper import MongoHelper as SqlHelper
else:
#免费和专用则存储在redis
from db.RedisHelper import RedisHelper as SqlHelper
#from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
#数据库初始化
sqlhelper.init_db()
return sqlhelper
except Exception as e:
raise Con_DB_Fail
# coding:utf-8
import sys
from config import DB_CONFIG
from util.exception import Con_DB_Fail
try:
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
except Exception as e:
raise Con_DB_Fail
def store_data(queue2, db_proxy_num):
'''
读取队列中的数据，写入数据库中
:param queue2:
:return:
'''
successNum = 0
failNum = 0
while True:
try:
proxy = queue2.get(timeout=300)
if proxy:
sqlhelper.insert(proxy)
# coding:utf-8
import sys
from config import DB_CONFIG
from util.exception import Con_DB_Fail
try:
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
elif DB_CONFIG['DB_CONNECT_TYPE'] == 'redis':
from db.RedisHelper import RedisHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
except Exception as e:
raise Con_DB_Fail
def store_data(queue2, db_proxy_num):
'''
读取队列中的数据，写入数据库中
:param queue2:
:return:
'''
successNum = 0
failNum = 0
while True:
try:
proxy = queue2.get(timeout=300)
if proxy:
for i in range(0, len(HEAD_BASE_PHONE)):
for j in range(0000, 9999):
phoneNum = obtainPhoneNum(HEAD_BASE_PHONE[i], j)
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
pipe = myredis.pipeline()
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
if __name__ == "__main__":
import redis
myredis = redis.Redis(db=0, host='127.0.0.1', port=6379)
mongo = MongoHelper()
mongo.init_db()
while (True):
phoneNum = myredis.rpop('phoneNum')
print(phoneNum)
try:
result = getProvider(phoneNum)
print(result)
except IndexError, e:
print(IndexError, e)
pipe = myredis.pipeline()
myredis.lpush("phoneNum", phoneNum)
pipe.execute()
continue
except urllib2.URLError, e:
print(urllib2.URLError, e)
continue