def test_save_ip(): p1 = ProxyIP(ip='192.168.0.1', port=443, provider='Foo', latency=200, stability=0.5) save_ip(p1) # basically the same ip p2 = ProxyIP(ip='192.168.0.1', port=443, provider='Foo', latency=200, stability=0.5) save_ip(p2) count = ProxyIP.select().where(ProxyIP.ip == '192.168.0.1').count() assert count == 1 p3 = ProxyIP(ip='192.168.0.1', port=80, provider='Foo', latency=200, stability=0.5) save_ip(p3) count = ProxyIP.select().where(ProxyIP.ip == '192.168.0.1').count() assert count == 2 ProxyIP.delete().execute()
async def api_v1_stats(request: Request): median_query: ProxyIP = ProxyIP.raw("""SELECT latency FROM proxy_ips WHERE is_valid = 1 ORDER BY latency LIMIT 1 OFFSET ( SELECT COUNT(*) FROM proxy_ips WHERE is_valid = 1 ) / 2""").get() median = median_query.latency mean_query: ProxyIP = ProxyIP.raw("""SELECT AVG(latency) as latency FROM proxy_ips WHERE is_valid = 1 AND latency < 9999""").get() mean = mean_query.latency valid_count = _get_valid_proxies_query().count() total_count = ProxyIP.select().count() return json({ 'median': median, 'valid_count': valid_count, 'total_count': total_count, 'mean': mean, })
def test_create_ip(): ip_str = create_test_ip() count = ProxyIP.select().count() assert count > 0 delete_test_ip(ip_str)
def feed_from_db(): # TODO: better query (order by attempts) proxies = ProxyIP.select().where(ProxyIP.updated_at > datetime.now() - timedelta(days=14)) for p in proxies: scheduler.validator_queue.put(p) logger.debug('Feed {} proxies from the database for a second time validation'.format(len(proxies)))
def get_proxy(https=False) -> ProxyIP: proxies: [ProxyIP] = ProxyIP.select().where(ProxyIP.is_valid == True).where(ProxyIP.stability >= 0.9) if https: proxies = proxies.where(ProxyIP.is_https == True) proxies = proxies.order_by(ProxyIP.updated_at.desc()).limit(63) proxy: ProxyIP = random.choice(proxies) return proxy
def save_ip(p: ProxyIP): basic_query = ProxyIP.select().where(ProxyIP.ip == p.ip) count = basic_query.count() if count == 0: # logger.debug('Creating new ip record: ' + p.__str__()) p.save() else: # logger.debug('Update an existing ip record: ' + p.__str__()) existing_proxy: ProxyIP = ProxyIP.get(ProxyIP.ip == p.ip) existing_proxy.assign_from(p) existing_proxy.save()
def save_ip(p: ProxyIP): basic_query = ProxyIP.select().where(ProxyIP.ip == p.ip) count = basic_query.count() if count == 0: logger.debug('Creating new ip record: ' + p.__str__()) p.save() else: logger.debug('Update an existing ip record: ' + p.__str__()) ProxyIP.update(latency=p.latency, stability=p.stability, is_valid=p.is_valid, is_anonymous=p.is_anonymous, updated_at=datetime.datetime.now()).where( ProxyIP.ip == p.ip).execute() logger.debug('Saved: ' + p.__str__())
def _get_valid_proxies_query(): return ProxyIP.select().where(ProxyIP.latency > 0).where(ProxyIP.latency < 9999) \ .where(ProxyIP.is_valid == True)