def check(self, proxy: ProxyModel):
if proxy.protocol == 'https':
judge = self.https_judge
else:
judge = self.http_judge
content = None
connector = proxy.connector
try:
start = time.time()
connector.connect() # connect to proxy
proxy.speed = int((time.time() - start) * 1000)
connector.negotiate(judge) # proxy connect to test web
request, request_headers = judge.build_request(self._method, fullpath=connector.use_full_path) # build test request
connector.send(request) # send request
content = connector.recv() # fetch response
judge.parse_response(proxy, content, request_headers) # parse response body and verify anonymity of proxy
except (socket.gaierror, RecvTimeout, ConnectTimeout, ConnectionResetError, IndexError, SSLError, ValueError, HTTPParseError, ConnectionAbortedError, BadResponseError, ProxySendError) as e:
proxy.exception = e
except BadStatusError as e:
proxy.exception = e
except JSONDecodeError:
logger.error('Json decode error when using %s' % proxy)
logger.debug(content)
except Exception as e:
proxy.exception = e
logger.error('Unexpected exception: %s, exception type: %s' % (proxy, type(e)))
logger.exception(e)
proxy.verified_at = datetime.datetime.now()
logger.debug('%s verified' % proxy)
return proxy
async def check(self, proxy: ProxyModel):
logger.debug('Start Verify Proxy: %s' % proxy)
proxy.usable = False
judge = self.https_judge if proxy.protocol == 'https' else self.http_judge
content = None
conn = Connector(proxy.ip, proxy.port)
negotiator = NGTRS[proxy.protocol.upper()](conn)
try:
await conn.connect()
await negotiator.negotiate(judge)
headers, content, hdrs = await _send_test_request(
self._method, conn, negotiator, judge)
content = _decompress_content(headers, content)
judge.parse_response(proxy, content, hdrs)
except ProxyError:
pass
except JSONDecodeError:
# Config.logger.error('Json decode error when using %s' % proxy)
logger.debug(content)
except Exception as e:
logger.error('Error: %s, type: %s' % (proxy, type(e)),
exc_info=True)
finally:
conn.close()
proxy.verified_at = datetime.datetime.now()
self.finished += 1
logger.debug('Finished Proxy: %s' % (proxy))
return proxy
def __remove_exist_proxies(pm, proxies):
all_proxy = set()
for i in pm.all_proxy():
all_proxy.add('%s:%d' % (i.ip, i.port))
result = []
for i in proxies:
if i not in all_proxy:
result += [
ProxyModel.from_url('%s://%s' % (j, i))
for j in ['http', 'https', 'socks5']
]
logger.info('Remove %d exist proxies' % (len(proxies) - len(result) / 3))
return result
def parse_response(self, proxy: ProxyModel, resp: str, hdrs):
resp = json.loads(resp)
proxy.usable = True
proxy.ip_feedback = resp['origin']
ret_header = resp['headers']
extra_headers = {}
for x in ret_header:
if x.lower() in ['host']:
continue
if x in hdrs and ret_header[x] == hdrs[x]:
continue
extra_headers[x] = ret_header[x]
if len(extra_headers) == 0:
proxy.anonymity = int(Anonymity.Elite)
else:
proxy.extra_headers = str(extra_headers)
__forwarded = 'X-Forwarded-For'
if __forwarded in extra_headers:
if self.ip in ret_header[__forwarded]:
proxy.anonymity = int(Anonymity.Transparent)
elif proxy.ip in ret_header[__forwarded]:
proxy.anonymity = int(Anonymity.Anonymous)
else:
proxy.anonymity = int(Anonymity.Confuse)
from manager import ProxyManager
from model import ProxyModel
if __name__ == '__main__':
# Config.Base.metadata.drop_all(Config.engine)
# Config.Base.metadata.create_all(Config.engine)
pm = ProxyManager()
pm.add_proxy(ProxyModel.instance('http://27.208.25.190:8060'))
def verify_failed(self, proxy: ProxyModel):
proxy.failed += 1
if proxy.success - proxy.failed > 15:
proxy.verifiable = 0
# coding:utf-8
import asyncio
import time
from model import ProxyModel
from proxy_verifier import ProxyAsyncVerifier
if __name__ == '__main__':
loop = asyncio.get_event_loop()
start = time.time()
proxy = ProxyModel.from_url('socks5://127.0.0.1:1080')
task = asyncio.ensure_future(ProxyAsyncVerifier(loop).check(proxy))
loop.run_until_complete(task)
print(time.time() - start)
print(task.result())
print(proxy.usable)
from model import ProxyModel
from manager import ProxyManager
if __name__ == '__main__':
# Config.Base.metadata.drop_all(Config.engine)
# Config.Base.metadata.create_all(Config.engine)
pm = ProxyManager()
pm.add_proxy(ProxyModel.from_url('http://27.208.25.190:8060'))