async def enrich(ua, **params):
"""
Detect device type using User-Agent string
https://github.com/selwin/python-user-agents
"""
try:
parsed = parse(ua)
res = Prodict(
os_family=parsed.os.family,
os_version=list(v if isinstance(v, int)
else 0 for v in parsed.os.version),
browser_family=parsed.browser.family,
browser_version=list(v if isinstance(
v, int) else 0 for v in parsed.browser.version,),
device_family=parsed.device.family,
device_brand=parsed.device.brand,
device_model=parsed.device.model)
res.is_bot = int(parsed.is_bot)
res.is_tables = int(parsed.is_tablet)
res.is_mob = int(parsed.is_mobile)
res.is_pc = int(parsed.is_pc)
return res
except Exception:
logger.exception('handle ex')
return response.error()
async def images_loader(self):
while True:
await asyncio.sleep(15)
try:
await image_navigator.load()
except Exception:
logger.exception('ex')
async def ws_sender(ws):
subscription = state.logs_reader()
while True:
try:
msg = await subscription.get()
if msg is None:
break
tstr = datetime.datetime.utcfromtimestamp(
msg.ts / 1000).strftime("%m%d %H:%m:%S.%s")[:18]
data = {
'id': msg.id,
'cid': msg.cid,
'cname': msg.name,
'time': tstr,
'ts': msg.ts,
'source': msg.source,
'data': msg.message
}
await ws.send_str(ujson.dumps(data))
await rpc.notify('logs', 'write', msg=msg)
except CancelledError:
logger.debug('ws writer closed')
break
except Exception:
logger.exception('ex')
break
async def websocket_handler(request):
ws = web.WebSocketResponse()
try:
await ws.prepare(request)
sender = await scheduler.spawn(ws_sender(ws))
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
if msg.data == 'close':
await ws.close()
else:
# MSG handler
pass
# await ws.send_str(msg.data + '/answer')
elif msg.type == aiohttp.WSMsgType.ERROR:
print('ws connection closed with exception %s' %
ws.exception())
print('websocket connection closed')
except CancelledError:
logger.info('CancelledError fetched')
except Exception:
logger.exception('ex')
finally:
await sender.close()
return ws
async def chech(p, params):
asyncio.sleep(random()*1)
check = Prodict(success=0, region=p.cityRu)
check.update({k: p[k] for k in copyattrs})
try:
state.myiter()
async with aiohttp.ClientSession() as chs:
proxy_auth = aiohttp.BasicAuth(p.user, p.password)
hostport = f"http://{p.host.strip()}:{p.port}"
async with chs.get(test_url, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as pr:
check.responseCode = pr.status
if pr.status == 200:
check.success = 1
check.contentSize = len(await pr.text())
async with chs.get(check_ip, proxy=hostport, proxy_auth=proxy_auth, timeout=10) as ip_r:
if ip_r.status == 200:
check.extIp = (await ip_r.text()).strip()
state.succ()
except ClientConnectorError:
logger.warn('connection error: %s:%s', p.host, p.port)
except asyncio.TimeoutError:
logger.warn('asyncio timeout')
except TimeoutError:
logger.warn('timeout')
except Exception:
logger.exception('check err')
await asyncio.sleep(1)
try:
async with aiohttp.ClientSession() as ss:
async with ss.post(params.notify, json=check, timeout=10) as wh_r:
if wh_r.status != 200:
logger.error('webhook failed')
except Exception:
logger.exception('sending webhook err')
async def remove_container(self, name):
# removing if running
try:
container = await self.get(name)
if container:
await container.fill()
if container.state == 'running':
container_autoremove = container.auto_removable()
logger.info("Stopping container")
await container.stop()
if not container_autoremove:
await container.delete()
else:
await container.delete()
await asyncio.sleep(0.5)
# try:
# await container.wait(condition="removed")
# except DockerError as e:
# logger.debug('Docker 404 received on wait request')
# if e.status != 404:
# raise e
except DockerError:
logger.exception('container remove exc')
return True
async def do_query(url, params):
try:
async with aiohttp.ClientSession() as s:
async with s.get(url, timeout=10, params=params) as r:
return await r.json()
except Exception:
logger.exception('err')
return RESULT_INTERNAL_ERROR
async def manager():
await register(settings.hook)
while True:
try:
await join_phase()
except Exception:
logger.exception('ex')
await asyncio.sleep(5)
async def download_db():
try:
if not os.path.isfile(TRG):
logger.info('downloading database. cmd: %s', CMD)
out = subprocess.call(CMD, shell=True)
logger.info('download result %s', out)
state['geodata'] = maxminddb.open_database(TRG)
except Exception:
logger.exception('download err')
async def get(ip, **params):
try:
if 'geodata' in state:
location = state['geodata'].get(ip)
return location
return {'result': RESULT_NOT_LOADED_YET}
except Exception:
logger.exception('get ip err')
return {'result': RESULT_INTERNAL_ERROR}
async def service_worker():
for num in count():
try:
if num == 0:
pass
except asyncio.CancelledError:
break
except Exception:
logger.exception('exc')
await asyncio.sleep(30)
async def startup():
try:
logger.info('executing: %s', CMD)
out = subprocess.call(CMD, shell=True)
logger.info('download result: %s', out)
gl = state['geodata'] = GeoLocator(TRG, MODE_BATCH | MODE_MEMORY)
logger.info('DB version %s (%s)', gl.get_db_version(),
gl.get_db_date())
except Exception:
logger.exception('download err')
async def enrich(ip, **params):
if state.geodata:
try:
location = state.geodata.get_location(ip, detailed=True)
if location and 'info' in location:
return handle_location(**pdict.from_dict(location['info']))
return response.error('Database not ready')
except Exception:
logger.exception('mmgeo error')
return response.error('Error while quering database')
async def events_stat(**params):
events_where = stat_queries.events_where()
query = stat_queries.events(events_where) + stat_queries.FMT_JSON
try:
async with timeout(1):
stat_events = await ch.select(query)
return ujson.loads(stat_events)['data'] if stat_events else []
except asyncio.TimeoutError:
logger.exception('stat get error')
except asyncio.CancelledError:
pass
return []
async def enrich(**params):
try:
ip = params.get('ip', None)
if not state.db:
raise HTTPServiceUnavailable()
if ip:
location = state.db.get(ip)
if location:
return handle_location(**location)
return {}
except Exception:
logger.exception('mmgeo error.', location=location)
return response.error()
async def startup():
"""
Load database on startup
"""
try:
if not os.path.isfile(settings.db_file):
raise FileNotFoundError("db file not found")
state.geodata = GeoLocator(settings.db_file,
MODE_BATCH | MODE_MEMORY)
logger.info('DB version', dbver=state.geodata.get_db_version(),
dbdate=state.geodata.get_db_date())
except Exception:
logger.exception('error while opening database file')
async def clean_worker(self):
while True:
# Remove expired services
try:
await asyncio.sleep(5)
await self.resolve_docstatus_all()
await self.check_regs_changed()
except ConnectionRefusedError:
logger.error('Redis connection refused')
except asyncio.CancelledError:
logger.warn('Asyncio cancelled')
break
except Exception:
logger.exception('state initialize')
await asyncio.sleep(1)
async def loader():
try:
if not os.path.isfile(settings.db_file):
raise FileNotFoundError("db file not found")
state.db = maxminddb.open_database(settings.db_file)
logger.info('DB loaded')
for num in count():
info = await cache_info()
logger.info('cache stat', loop=num, info=info)
await asyncio.sleep(60 * 5)
except asyncio.CancelledError:
pass
except Exception:
logger.exception('error while opening database file')
async def lease_cleaner():
while True:
try:
for uid in [*state.users.keys()]:
now = ms()
user = state.users[uid]
# Has required attrtibutes
if user.act and user.start:
# Less then 5 min from last activity
if now - user.act < 60*5*1000:
# Less then 5 min from lease
if now - user.start < 60*15*1000:
continue
await state.rm_user(uid)
except Exception:
logger.exception('lease_cleaner ex')
await asyncio.sleep(5)
async def error_middleware(request, handler):
try:
response = await handler(request)
return response
except HTTPException as ex:
logger.exception('error middleware http ex')
return json_response({'error': ex.reason},
status=ex.status,
request=request)
except Exception as ex:
logger.exception('error middleware ex')
return json_response({'error': 'Internal server error'},
status=500,
request=request)
return error_middleware
async def download_db():
while True:
try:
await app['rpool'].subscribe(app['mpsc'].channel('any'))
logger.info('subscribed any')
async with aiohttp.ClientSession() as session:
while True:
batch = state.grab()
if len(batch):
enc = json.dumps(batch, ensure_ascii=False)
q = urllib.parse.urlencode(
{'data': base64.encodebytes(enc.encode())})
async with session.post(MP_TRACK_EP, data=q) as resp:
logger.info('uploading %s items. code %s',
len(batch), resp.status)
await asyncio.sleep(1)
pass
except Exception:
logger.exception('err - root loop')
async def checker():
scheduler = await aiojobs.create_scheduler(limit=CONCURRENT_CHECKS)
while True:
state.myloop()
try:
params = settings.proxy_checker
headers = {"Authorization": params.auth}
async with aiohttp.ClientSession(headers=headers) as s:
async with s.get(params.list, timeout=5) as r:
for proxy in await r.json():
p = Prodict.from_dict(proxy)
await scheduler.spawn(chech(p, params))
except Exception:
logger.exception('err')
jobs = scheduler._jobs
while True:
await asyncio.sleep(0.1)
if len(jobs) == 0:
logger.info('finished')
break
async def uploader():
while True:
try:
async with aiohttp.ClientSession() as session:
while True:
if len(state.buffer):
buff = state.buffer.copy()
state.buffer.clear()
enc = ujson.dumps(buff, ensure_ascii=False)
q = urllib.parse.urlencode(
{'data': base64.encodebytes(enc.encode())})
async with session.post(settings.endpoint,
data=q) as resp:
logger.info('uploading',
items=len(buff),
status=resp.status)
await asyncio.sleep(1)
pass
except asyncio.CancelledError:
break
except Exception:
logger.exception('exc')
await asyncio.sleep(5)
async def request_handler(request, handler):
# get query
query = dict(**request.query)
# url params
if request.method == 'POST':
if request.content_type == 'application/json':
raw = await request.text()
if raw:
query.update(json_loads(raw))
else:
post = await request.post()
query.update(post)
# url params
query.update(request.match_info)
try:
result = await handler(**query)
# Handling stream responses
if isinstance(result, types.AsyncGeneratorType):
try:
stream = StreamResponse()
await stream.prepare(request)
async for block in result:
await stream.write(block)
await stream.write_eof()
return stream
# Halt handling
except asyncio.CancelledError:
logger.warn('halted response')
return stream
# regilar response
else:
return json_response(result, request=request)
except ReceivedErrorResponse as e:
return json_response(response.error(e.message), request=request)
except Exception as e:
logger.exception("exc")
return json_response(response.error(str(e)), request=request)
async def progress(**params):
try:
return {'result': state.myget()}
except Exception:
logger.exception('err')
return {'result': RESULT_INTERNAL_ERROR}