def getAll(self, https=False):
"""
get all proxy from pool as Proxy list
:return:
"""
proxies = self.db.getAll(https)
return [Proxy.createFromJson(_) for _ in proxies]
def testRedisClient():
from db.dbClient import DbClient
from helper.proxy import Proxy
uri = "redis://:[email protected]:6379"
db = DbClient(uri)
db.changeTable("use_proxy")
proxy = Proxy.createFromJson(
'{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}'
)
print("put: ", db.put(proxy))
print("get: ", db.get())
print("exists: ", db.exists("27.38.96.101:9797"))
print("exists: ", db.exists("27.38.96.101:8888"))
print("pop: ", db.pop())
print("getAll: ", db.getAll())
print("getCount", db.getCount())
def getAll(self):
"""
get all proxy from pool as Proxy list
:return:
"""
proxies_dict = self.db.getAll()
return [Proxy.createFromJson(value) for _, value in proxies_dict.items()]
def update_fail_count(self, proxy_str):
proxy = self.db.get(proxy_str)
if not proxy:
return None
proxy = Proxy.createFromJson(proxy)
proxy.fail_count += 1
self.db.update(proxy)
return proxy
def pop(self):
"""
return and delete a useful proxy
:return:
"""
proxy = self.db.pop()
if proxy:
return Proxy.createFromJson(proxy)
return None
def get(self):
"""
return a useful proxy
:return:
"""
proxy = self.db.get()
if proxy:
return Proxy.createFromJson(proxy)
return None
def getByTag(self, tag):
"""
return a useful proxy by tag
:return:
"""
proxy = self.db.getByTag(tag)
if proxy:
return Proxy.createFromJson(proxy)
return None
def get(self, https=False):
"""
return a proxy
Args:
https: True/False
Returns:
"""
proxy = self.db.get(https)
return Proxy.createFromJson(proxy) if proxy else None
def testProxyClass():
proxy = Proxy("127.0.0.1:8080")
print(proxy.to_json)
proxy.source = "test"
proxy_str = json.dumps(proxy.to_dict, ensure_ascii=False)
print(proxy_str)
print(Proxy.createFromJson(proxy_str).to_dict)
def getAll(self, tag=None):
"""
字典形式返回所有代理, 使用changeTable指定hash name
:return:
"""
sql = 'select url, score, proxy_type, tag from proxy'
if tag is not None:
sql += f' where tag="{tag}"'
df = pd.read_sql(sql, self.engine)
proxies = []
for index, row in df.iterrows():
proxy = Proxy.createFromJson(row.to_json())
proxies.append(proxy)
return proxies
def testProxyClass1():
proxy = Proxy("https://127.0.0.1:8080")
print(proxy.to_json)
proxy.score = 10
proxy.proxy_type = "test"
proxy.tag = "高匿"
proxy_str = json.dumps(proxy.to_dict, ensure_ascii=False)
print(proxy_str)
print(Proxy.createFromJson(proxy_str).to_dict)
def testRedisClient():
from db.dbClient import DbClient
from helper.proxy import Proxy
uri = "redis://:[email protected]:6379"
db = DbClient(uri)
db.changeTable("use_proxy")
proxy = Proxy.createFromJson(
'{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}')
# print("put: ", db.put(proxy))
# print("put: ", db.putTag(tag='test', proxy='1238:7'))
# print("del: ", db.deleteTag(tag='test', proxy='1234'))
print("get: ", db.getByTag(tag='test'))
def run(self):
self.log.info("ProxyCheck - {} : start".format(self.name))
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("ProxyCheck - {} : complete".format(self.name))
break
proxy = Proxy.createFromJson(proxy_json)
proxy = proxyCheck(proxy)
if self.type == "raw":
if proxy.last_status:
if self.proxy_handler.exists(proxy):
self.log.info('ProxyCheck - {} : {} exists'.format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info('ProxyCheck - {} : {} success'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info('ProxyCheck - {} : {} fail'.format(
self.name, proxy.proxy.ljust(23)))
else:
if proxy.last_status:
self.log.info('ProxyCheck - {} : {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.update(proxy)
else:
if proxy.fail_count > self.conf.maxFailCount:
self.log.info(
'ProxyCheck - {} : {} fail, count {} delete'.
format(self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
'ProxyCheck - {} : {} fail, count {} keep'.format(
self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.update(proxy)
self.queue.task_done()
def testRedisClient():
from db.dbClient import DbClient
from helper.proxy import Proxy
uri = "redis://:[email protected]:6379"
db = DbClient(uri)
db.changeTable("use_proxy")
proxy = Proxy.createFromJson('{"proxy": "118.190.79.36:8090", "https": false, "fail_count": 0, "region": "", "anonymous": "", "source": "freeProxy14", "check_count": 4, "last_status": true, "last_time": "2021-05-26 10:58:04"}')
print("put: ", db.put(proxy))
print("get: ", db.get(https=None))
print("exists: ", db.exists("27.38.96.101:9797"))
print("exists: ", db.exists("27.38.96.101:8888"))
print("pop: ", db.pop(https=None))
print("getAll: ", db.getAll(https=None))
print("getCount", db.getCount())