def get_db():
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
elif DB_CONFIG['DB_CONNECT_TYPE'] == 'redis':
from db.RedisHelper import RedisHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
return sqlhelper
def get_sqlhelper(proxy_type):
try:
#专用代理存储MongoDB
if DB_CONFIG[proxy_type] == 'DEDICATED':
from db.MongoHelper import MongoHelper as SqlHelper
else:
#免费和专用则存储在redis
from db.RedisHelper import RedisHelper as SqlHelper
#from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
#数据库初始化
sqlhelper.init_db()
return sqlhelper
except Exception as e:
raise Con_DB_Fail
# coding:utf-8
import sys
from config import DB_CONFIG
from util.exception import Con_DB_Fail
try:
if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':
from db.MongoHelper import MongoHelper as SqlHelper
else:
from db.SqlHelper import SqlHelper as SqlHelper
sqlhelper = SqlHelper()
sqlhelper.init_db()
except Exception as e:
raise Con_DB_Fail
def store_data(queue2, db_proxy_num):
'''
读取队列中的数据，写入数据库中
:param queue2:
:return:
'''
successNum = 0
failNum = 0
while True:
try:
proxy = queue2.get(timeout=300)
if proxy:
sqlhelper.insert(proxy)
