def __init__(self):
"""
init
:return:
"""
self.config = GetConfig()
self.__initDbClient()
def __init__(self, host="localhost", client=None):
self.config = GetConfig()
self.client = MongoClient('mongodb://localhost:27017/') if client is None else client
self.ssdb_client = DbClient()
self.db = self.client.spider
if self.db.lazada.count() is 0:
self.db.lazada.create_index([("status", ASCENDING),
("pr", DESCENDING)])
def run():
config = GetConfig()
if sys.platform.startswith("win"):
app.run(host=config.host_ip, port=config.host_port)
else:
app.run(host=config.host_ip,
port=config.host_port,
threaded=False,
processes=config.processes)
def testGetConfig():
"""
test class GetConfig in Util/GetConfig
:return:
"""
gg = GetConfig()
print(gg.db_type)
print(gg.db_name)
print(gg.db_host)
print(gg.db_port)
assert isinstance(gg.proxy_getter_functions, list)
print(gg.proxy_getter_functions)
def testGetFreeProxy():
"""
test class GetFreeProxy in ProxyGetter/GetFreeProxy
:return:
"""
gc = GetConfig()
proxy_getter_functions = gc.proxy_getter_functions
for proxyGetter in proxy_getter_functions:
proxy_count = 0
for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
if proxy:
print('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
proxy_count += 1
assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
def __init__(self, name, host, port, password):
"""
init
:param name:
:param host:
:param port:
:return:
"""
self.config = GetConfig()
self.name = name
print self.config.db_host
print self.config.db_port
print self.config.db_password
self.__conn = redis.Redis(host=self.config.db_host,
port=self.config.db_port,
db='0',
password=self.config.db_password)
def __init__(self):
self.config = GetConfig()
self.__initClient()
def __init__(self,): super(mongodb, self).__init__() self.dbConfig=GetConfig() self.conn = MongoClient(host=self.dbConfig.db_host,port=self.dbConfig.db_port)
def run():
config = GetConfig()
app.run(host=config.host_ip,
port=config.host_port,
threaded=False,
processes=config.processes)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import requests, sys, time
sys.path.append('../')
from Util.GetConfig import GetConfig
ip = GetConfig().db_host
def get():
try:
ret = requests.get('http://' + ip + ':5010/get/').text.replace(
'<br>', '').replace(' ',
'').replace(' ', '').replace('\r', '').replace(
'\n', '').replace('\t', '')
return ret
except Exception as e:
return "error_message=" + str(e)
def delet(ip):
try:
ret = requests.get('http://' + ip +
':5010/delete/?proxy={}'.format(ip)).text.replace(
'<br>',
'').replace(' ', '').replace(' ', '').replace(
'\r', '').replace('\n',
'').replace('\t', '')
return ret
def __init__(self):
self.config = GetConfig()
self.client = sa.create_engine(self.config.click_url)
def __init__(self): super(db_mysql, self).__init__() self.config = GetConfig()
def run():
config = GetConfig()
app.run(host=config.host_ip, port=config.host_port)
def run():
config = GetConfig()
app.run(host=config.host_ip, port=config.host_port, processes=5)
from Manager.ProxyManager import ProxyManager
sys.path.append('../')
from Util.GetConfig import GetConfig
app = Flask(__name__)
api_list = {
'get': u'get an usable proxy',
'refresh': u'refresh proxy pool',
'get_all': u'get all proxy from proxy pool',
'delete?proxy=127.0.0.1:8080': u'delete an unable proxy',
}
mini_proxy_num = GetConfig().mini_proxy_num
@app.route('/')
def index():
return jsonify(api_list)
@app.route('/get/')
def get():
proxy = ProxyManager().get()
status = ProxyManager().get_status()
num = int(status.pop('useful_proxy'))
if num < mini_proxy_num:
#print 'NULL'
return u'NULL'
def inckey(self, key, value):
self.__conn.hincrby(self.name, key, value)
def getAll(self):
return self.__conn.hgetall(self.name).keys()
def get_status(self):
return self.__conn.hlen(self.name)
def changeTable(self, name):
self.name = name
if __name__ == '__main__':
gg = GetConfig()
print(gg.db_type)
print(gg.db_name)
print(gg.db_host)
print(gg.db_port)
print(gg.db_password)
redis_con = RedisClient(gg.db_name, gg.db_host, gg.db_port, gg.db_password)
redis_con.put('abc')
redis_con.put('123')
# redis_con.put('123.115.235.221:8800')
# redis_con.put(['123', '115', '235.221:8800'])
# print(redis_con.getAll())
# redis_con.delete('abc')
# print(redis_con.getAll())
def check_auth(username, password):
config = GetConfig()
return username == config.username and password == config.password
def __init__(self):
self.db = DbClient()
self.config = GetConfig()
self.raw_proxy_queue = 'raw_proxy'
self.useful_proxy_queue = 'useful_proxy_queue'
def __init__(self,): super(Redisdb, self).__init__() self.config = GetConfig()
def __init__(self):
self.db = DbClient()
self.config = GetConfig()
self.raw_proxy_queue = 'raw_proxy'
self.log = LogHandler('proxy_manager')
self.useful_proxy_queue = 'useful_proxy'
Change Activity:
2016/12/4:
-------------------------------------------------
"""
__author__ = 'JHao'
import sys
sys.path.append('../')
from flask import Flask, jsonify, request
from Util.GetConfig import GetConfig
from Manager.ProxyManager import ProxyManager
config = GetConfig()
host_token = config.host_token
app = Flask(__name__)
api_list = {
'get': u'get an usable proxy',
# 'refresh': u'refresh proxy pool',
'get_all': u'get all proxy from proxy pool',
'delete?proxy=127.0.0.1:8080': u'delete an unable proxy',
'get_status': u'proxy statistics'
}
@app.route('/')
def index():
return jsonify(api_list)