async def home(request):
api = {
"/": "this page",
"/one": "get one proxy",
"/all": "list all proxy",
}
try:
sess = sess_maker()
available_proxy = sess.query(Proxy).filter(
Proxy.status == STATUS_OK).count()
new_proxy = sess.query(Proxy).filter(
Proxy.status == STATUS_NEW).count()
error_proxy = sess.query(Proxy).filter(
Proxy.status == STATUS_ERROR).count()
sess.close()
return web.json_response({
"api": api,
"status": {
"available_proxy": available_proxy,
"new_proxy": new_proxy,
"error_proxy": error_proxy,
},
})
except:
return web.json_response({
"api": api,
"status": "sql error",
})
async def verify_error_proxy_task():
logger.info("run verify_error_proxy_task")
s = sess_maker()
c = s.query(Proxy).filter(Proxy.status == STATUS_OK).count()
s.close()
if c < VERIFY_ERROR_LIMIT:
await verify_error_proxy()
s = sess_maker()
c = s.query(Proxy).filter(Proxy.status == STATUS_ERROR).count()
if c > MAX_ERROR_PROXIES:
res = s.query(Proxy).filter(Proxy.status == STATUS_ERROR).order_by(
asc(Proxy.updated_at)).limit(c -
MAX_ERROR_PROXIES).from_self().all()
[s.delete(i) for i in res]
s.commit()
def callback(f):
sess = sess_maker()
for p in f.result():
try:
sess.query(Proxy).filter(Proxy.ip_port == p.ip_port).one()
except sqlalchemy.orm.exc.NoResultFound:
if is_ip_port(p.ip_port):
sess.add(p)
sess.commit()
async def verify_all_proxy():
s = sess_maker()
proxies = s.query(Proxy).all()
s.close()
tasks = list()
for p in proxies:
tasks.append(asyncio.ensure_future(verify_and_update(p)))
if tasks:
await asyncio.wait(tasks)
async def verify_error_proxy():
s = sess_maker()
proxies = s.query(Proxy).filter(Proxy.status == STATUS_ERROR).all()
s.close()
tasks = list()
for p in proxies:
tasks.append(asyncio.ensure_future(verify_and_update(p)))
if tasks:
await asyncio.wait(tasks)
def update_proxy_status(p: Proxy, status):
logger.debug(f"update {p.ip_port} {p.status} to {status}")
if p.status != status:
s = sess_maker()
s.query(Proxy).filter(Proxy.ip_port == p.ip_port).update({'status': status})
s.commit()
def get_all_proxy():
s = sess_maker()
res = s.query(Proxy).filter(Proxy.status == STATUS_OK).all()
s.close()
return res
def get_one_proxy():
s = sess_maker()
res = s.query(Proxy).filter(Proxy.status == STATUS_OK).order_by(func.random()).limit(1).one()
s.close()
return res
def new_proxy_gauge_fn():
s = sess_maker()
res = s.query(Proxy).filter(Proxy.status == STATUS_NEW).count()
s.close()
return res
def error_proxy_gauge_fn():
s = sess_maker()
res = s.query(Proxy).filter(Proxy.status == STATUS_ERROR).count()
s.close()
return res
def available_proxy_gauge_fn():
s = sess_maker()
res = s.query(Proxy).filter(Proxy.status == STATUS_OK).count()
s.close()
return res
async def update_squid_task():
logger.info("run update_squid_task")
s = sess_maker()
proxies = s.query(Proxy).filter(Proxy.status == STATUS_OK).all()
s.close()
squid.update_conf(proxies)