def test_save_ip():
p1 = ProxyIP(ip='192.168.0.1',
port=443,
provider='Foo',
latency=200,
stability=0.5)
save_ip(p1)
# basically the same ip
p2 = ProxyIP(ip='192.168.0.1',
port=443,
provider='Foo',
latency=200,
stability=0.5)
save_ip(p2)
count = ProxyIP.select().where(ProxyIP.ip == '192.168.0.1').count()
assert count == 1
p3 = ProxyIP(ip='192.168.0.1',
port=80,
provider='Foo',
latency=200,
stability=0.5)
save_ip(p3)
count = ProxyIP.select().where(ProxyIP.ip == '192.168.0.1').count()
assert count == 2
ProxyIP.delete().execute()
async def api_v1_stats(request: Request):
median_query: ProxyIP = ProxyIP.raw("""SELECT latency
FROM proxy_ips
WHERE is_valid = 1
ORDER BY latency
LIMIT 1
OFFSET (
SELECT COUNT(*) FROM proxy_ips WHERE is_valid = 1
) / 2""").get()
median = median_query.latency
mean_query: ProxyIP = ProxyIP.raw("""SELECT AVG(latency) as latency
FROM proxy_ips
WHERE is_valid = 1 AND latency < 9999""").get()
mean = mean_query.latency
valid_count = _get_valid_proxies_query().count()
total_count = ProxyIP.select().count()
return json({
'median': median,
'valid_count': valid_count,
'total_count': total_count,
'mean': mean,
})
def test_create_ip():
ip_str = create_test_ip()
count = ProxyIP.select().count()
assert count > 0
delete_test_ip(ip_str)
def feed_from_db():
# TODO: better query (order by attempts)
proxies = ProxyIP.select().where(ProxyIP.updated_at > datetime.now() - timedelta(days=14))
for p in proxies:
scheduler.validator_queue.put(p)
logger.debug('Feed {} proxies from the database for a second time validation'.format(len(proxies)))
def get_proxy(https=False) -> ProxyIP:
proxies: [ProxyIP] = ProxyIP.select().where(ProxyIP.is_valid == True).where(ProxyIP.stability >= 0.9)
if https:
proxies = proxies.where(ProxyIP.is_https == True)
proxies = proxies.order_by(ProxyIP.updated_at.desc()).limit(63)
proxy: ProxyIP = random.choice(proxies)
return proxy
def save_ip(p: ProxyIP):
basic_query = ProxyIP.select().where(ProxyIP.ip == p.ip)
count = basic_query.count()
if count == 0:
# logger.debug('Creating new ip record: ' + p.__str__())
p.save()
else:
# logger.debug('Update an existing ip record: ' + p.__str__())
existing_proxy: ProxyIP = ProxyIP.get(ProxyIP.ip == p.ip)
existing_proxy.assign_from(p)
existing_proxy.save()
def save_ip(p: ProxyIP):
basic_query = ProxyIP.select().where(ProxyIP.ip == p.ip)
count = basic_query.count()
if count == 0:
logger.debug('Creating new ip record: ' + p.__str__())
p.save()
else:
logger.debug('Update an existing ip record: ' + p.__str__())
ProxyIP.update(latency=p.latency,
stability=p.stability,
is_valid=p.is_valid,
is_anonymous=p.is_anonymous,
updated_at=datetime.datetime.now()).where(
ProxyIP.ip == p.ip).execute()
logger.debug('Saved: ' + p.__str__())
def _get_valid_proxies_query():
return ProxyIP.select().where(ProxyIP.latency > 0).where(ProxyIP.latency < 9999) \
.where(ProxyIP.is_valid == True)
