def __index_modules(reload=False):
"""Add all modules in module_list."""
global __modules_data
if __modules_data and not reload:
return
dir_list = sorted(os.listdir(MODULES_PATH))
nb = 0
__modules_data.clear()
for module_name in dir_list:
if is_disabled(module_name):
continue
if '__pycache__' in module_name:
continue
__modules_data[module_name] = {
'disabled': False,
'instance': None,
'thread': None
}
Log.debug('index "%s" module' % module_name)
nb += 1
Log.info('%d modules indexed' % nb)
class UpdateToken:
def __init__(self, config_vk):
self.log = Log("UpdateToken")
self.config = config_vk
self.client_id = self.config.client_id
self.finished = Event()
async def __call__(self) -> str:
async with RedirectServer() as server:
redirect_address = server.redirect_address
url = "https://oauth.vk.com/authorize" \
f"?client_id={self.client_id}" \
"&display=page" \
f"&redirect_uri={redirect_address}" \
"&scope=friends,wall,offline,groups" \
"&response_type=token" \
"&v=5.103"
webbrowser.open_new(url)
data = await server()
self.config.token = data['access_token']
self.config.user_id = data['user_id']
self.config.update()
self.log.info("Token updated")
self.finished.set()
async def x(index):
log = Log(f"{from_}=>{to_}#{index}")
retries = retries_
while True:
found = False
async for item in db.choose(M, {FIELD_NAME: from_}, {FIELD_NAME: to_}, limit_=limit):
log.info(item=item)
processed[(from_, to_)] += 1
await asyncio.sleep(sleep_time)
found = True
if found:
break
if retries == 0:
return
retries -= 1
log.warning(retry=retries_ - retries)
await asyncio.sleep(sleep_time * sleep_coef)
log.important("STOP")
def stop_all():
"""Stop all modules."""
nb = 0
for module_name in __modules_data.copy():
if __modules_data[module_name]['instance'] and stop(module_name):
nb += 1
Log.info('%d modules stopped' % nb)
def load_config(module_name):
"""Load module config."""
if is_disabled(module_name):
return False
if call(module_name, 'load_config', c_optional_call=True):
Log.info('start "%s" module' % module_name)
return True
return False
def stop(module_name):
"""Stop module."""
__active_modules_name.remove(module_name)
Log.info('stop "%s" module' % module_name)
__modules_data[module_name]['instance'].stopped()
__modules_data[module_name]['instance'].is_running = False
__modules_data.pop(module_name)
return True
def stop():
"""Stop daemon."""
global __is_running
if not __is_running:
return
Log.info('stop deamon')
__is_running = False
ModuleManager.stop_all()
sys.exit(0)
class RedirectServer:
def __init__(self):
self.log = Log("RedirectServer")
self.app: web.Application = None
self.runner: web.AppRunner = None
self.site: web.TCPSite = None
self.template: str = None
self.data = None
self.data_received = Event()
self.address = "localhost"
self.port = randint(8081, 9000)
@property
def redirect_address(self):
return f"{self.address}:{self.port}/redirect"
async def warm_up(self):
async with aiofiles.open("vk_utils/redirect_page.html", mode='rt') as f:
self.template = await f.read()
self.app = web.Application()
self.app.add_routes([
web.get('/redirect', self._handler_get),
web.post('/redirect', self._handler_post)
])
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.site = web.TCPSite(self.runner, self.address, self.port)
self.log.info("Start redirect server")
await self.site.start()
async def _handler_get(self, request: BaseRequest):
return web.Response(body=self.template, content_type="text/html")
async def _handler_post(self, request: BaseRequest):
path = (await request.json())['answer']
self.data = dict(item.split('=') for item in path.split("&"))
self.data_received.set()
return web.json_response({'status': 'ok'})
async def __call__(self):
await self.data_received.wait()
self.log.info("Data received!")
return self.data
async def __aenter__(self):
await self.warm_up()
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
self.log.info("Stopping WebServer")
await self.site.stop()
await self.runner.cleanup()
await self.app.cleanup()
self.log.info("WebServer stopped")
def LoadConfig(file_name: str = None) -> "Config":
log = Log("LoadConfig")
file_name = file_name or "private.yaml"
log.info("Load config", source=file_name)
with open(file_name, "rt") as f:
data = yaml.safe_load(f)
def do_update():
log.info("Update config file", source=file_name)
with open(file_name, "wt") as f:
yaml.safe_dump(data, f)
return Config(data, do_update, "", source=file_name)
def start(module_name):
"""Start module."""
if is_disabled(module_name):
return False
if module_name not in __modules_data or __modules_data[module_name]['instance'] is None:
return False
__modules_data[module_name]['instance'].started()
__modules_data[module_name]['instance'].is_running = True
__modules_data[module_name]['thread'] = threading.Thread(
None,
__modules_data[module_name]['instance'].run,
'module: %s' % module_name
)
__modules_data[module_name]['thread'].setDaemon(True)
__modules_data[module_name]['thread'].start()
__active_modules_name.append(module_name)
Log.info('start "%s" module' % module_name)
return True
def add_info(self, text, *args, **kwargs):
"""Add a message to logger"""
text = '%s %s: %s' % (self.name_prefix, self.name, text)
Log.info(text, *args, **kwargs)
def load_config_all():
"""Load all modules config."""
started = [load_config(module_name) for module_name in __modules_data]
Log.info('%d modules config loaded' % sum(started))
class Monitoring:
"""Класс веб-сервера"""
API_PATH = "/api"
def __init__(self, addr, port):
self.addr = addr
self.port = port
self.log = Log(f"Monitoring")
self._pages: Dict[str, BasePage] = {}
self.main_page = MainPage("_main", "Info")
self.add_page(self.main_page)
self.app: web.Application = None
self._middleware = []
def __getitem__(self, item: str) -> BasePage:
"""Возвращает страницу"""
return self._pages[item]
def __getattr__(self, item):
try:
return super().__getattr__(item)
except AttributeError:
return self._pages[item]
def add_page(self, page: BasePage):
assert page.id not in self._pages
self._pages[page.id] = page
def add_middleware(self, func):
self._middleware.append(func)
async def warm_up(self):
self.log.info("Running", addr=self.addr, port=self.port)
self.log.important("You can access to monitoring by",
addr=f"http://{self.addr}:{self.port}/")
self.app = web.Application(middlewares=[self.error_middleware])
self.app.add_routes([
web.get(f"/", self._get_index),
web.get(f"{self.API_PATH}/ping", self._get_ping),
web.get(f"{self.API_PATH}/pages", self._get_pages),
web.get(f"{self.API_PATH}/page/{{id}}", self._get_page),
])
self.runner = web.AppRunner(self.app)
await self.runner.setup()
site = web.TCPSite(self.runner, self.addr, self.port)
await site.start()
self.log.info("Server started")
self.main_page.start_time = time()
async def shutdown(self):
self.log.info("Server shutdown")
await self.runner.shutdown()
self.log.info("Server cleanup")
await self.runner.cleanup()
self.log.info("Server gracefully stopped")
async def _get_index(self, request):
async with aiofiles.open("core/monitor/app.html", mode='rt') as f:
template = await f.read()
return web.Response(body=template, content_type="text/html")
async def _get_ping(self, request):
return web.json_response({'status': 'ok', 'time': time()})
async def _get_pages(self, request):
answer_ = []
for page in self._pages.values():
answer_.append(page.page_info())
return web.json_response(answer_)
async def _get_page(self, request):
page_id = request.match_info['id']
# TODO: NotFoundPage
page = self._pages[page_id]
return web.json_response(page.to_dict())
@web.middleware
async def error_middleware(self, request: Request, handler):
try:
for md in self._middleware:
md(request, handler)
self.main_page.queries += 1
response = await handler(request)
if response.status == 200:
return response
message = response.message
except web.HTTPException as ex:
self.log.exception(request=request, handler=handler)
if ex.status != 404:
raise
message = ex.reason
except Exception as e:
self.log.exception(request=request, handler=handler)
message = str(e)
return web.json_response({'status': 'error', 'message': message})
class TimeSeries:
def __init__(self, name: str, ts: InpT, vs: Optional[InpT] = None):
self.name = name
self.log = Log(f"TimeSeries:{name}")
if vs is None:
self.ts, self.vs = np.unique(ts, return_counts=True)
else:
self.ts, self.vs = ts, vs
assert isinstance(self.ts, np.ndarray)
assert isinstance(self.vs, np.ndarray)
assert len(self.ts) == len(self.vs)
@staticmethod
def _normalize(t):
if t is None:
return t
if isinstance(t,
(int, float, np.int64, np.uint, np.integer, np.float)):
return t
if isinstance(t, datetime.datetime):
return t.timestamp()
if isinstance(t, datetime.timedelta):
return t.total_seconds()
raise TypeError(type(t))
def __getitem__(self, item: slice) -> "TimeSeries":
start = self._normalize(item.start or self.ts.min())
stop = self._normalize(item.stop or self.ts.max())
step = self._normalize(item.step)
if item.start or item.stop:
filter_ = (start <= self.ts) & (self.ts <= stop)
t, v = self.ts[filter_], self.vs[filter_]
else:
t, v = self.ts, self.vs
if not step:
return TimeSeries(f"{self.name}[sliced]", t, v)
count = int((stop - start) / step) or 1
xi = np.linspace(start, stop, num=count, dtype=int)
new_v = np.zeros((len(xi) + 1, ), dtype=int)
indexes = ((t - start) // step).round().astype(int)
np.add.at(new_v, indexes, v)
new_v[-2] += new_v[-1]
return TimeSeries(f"{self.name}[sliced#]", xi, new_v[:-1])
def min(self):
index = self.vs.argmin()
return self.ts[index], self.vs[index]
def max(self):
index = self.vs.argmax()
return self.ts[index], self.vs[index]
def sum(self):
return self.vs.sum()
def cumsum(self):
return TimeSeries(f"∫({self.name})", self.ts, self.vs.cumsum())
def __truediv__(self, other: float):
return TimeSeries(f"{self.name} / {other}", self.ts, self.vs / other)
def dist(self, ts: "TimeSeries"):
assert np.array_equal(
self.ts, ts.ts
), "Need identical grid for dist calculation. Use `.dist_grid`"
dist = np.sum((self.vs - ts.vs)**2)**0.5
return dist
def dist_grid(self, ts: "TimeSeries"):
"""Сравнивает, предварительно приведя к одной решетке"""
tss = np.unique(np.append(self.ts, ts.ts))
tss.sort()
start = tss.min()
stop = tss.max()
step = np.percentile(np.diff(tss), 50)
self.log.info("Grid:", start, stop, step)
self_g = self[start:stop:step]
ts_g = ts[start:stop:step]
self_g.dist(ts_g)
def p(self, percentile: float) -> float:
"""Перцентиль от значений"""
assert 0 < percentile < 100
return np.percentile(self.vs, percentile)
def peaks(self, percentile=95):
""" Возвращает локальные максимумы выше перцентиля """
indices, _ = find_peaks(self.vs, height=self.p(percentile))
for ts, vs in zip(self.ts[indices], self.vs[indices]):
yield int(ts), float(vs)
@staticmethod
def fmt_date(time_stamp: int):
return datetime.datetime.utcfromtimestamp(time_stamp).strftime(
'%Y-%m-%dT%H:%M:%SZ')
def __len__(self):
return len(self.ts)
def __eq__(self, other: "TimeSeries"):
return np.array_equal(self.ts, other.ts) and np.array_equal(
self.vs, other.vs)
def __repr__(self):
from_ = self.ts.min()
to_ = self.ts.max()
return f"<TimeSeries of `{self.name}` " \
f"[{self.fmt_date(from_)} - {self.fmt_date(to_)}] " \
f"{len(self.ts)} items, " \
f"∑={self.vs.sum():.2f}" \
f">"
def restart(module_name):
"""Restart module."""
Log.info('restart "%s" module' % module_name)
stop(module_name)
start(module_name)
def __server_ready(self, server, bind):
Log.info('Start MoLa network : %s:%d' % bind)
def __server_peer_connect(self, sock, host, port):
Log.info('Peer connect : %s' % host)
def start_all():
"""Start all modules."""
started = [start(module_name) for module_name in __modules_data]
Log.info('%d modules started' % sum(started))
def __server_peer_disconnect(self, sock):
Log.info('Peer disconect : %s' % str(sock))
class BaseWork:
start_time = time()
MUTE_EXCEPTION = True
PARALLEL = 10
INPUT_RETRIES = 0
WAIT_COEF = 1
need_stop = False
work_ids: Dict[str, int] = {}
def __init__(self):
work_name = self.__class__.__name__
self.log = Log(work_name)
self.log.debug("Register work")
self.work_ids.setdefault(work_name, -1)
self.work_ids[work_name] += 1
work_id = f"{work_name}_{self.work_ids[work_name]}"
self.log.debug("Work registered", work_id=work_id)
self.stat = Stats(work_id, work_name)
self.log.debug("Run task manager")
self.task_manager = TasksManager(self.PARALLEL)
self.tasks: List[TaskInfo] = []
self.state = "Base class initialized"
@property
def state(self):
return self.stat.state
@state.setter
def state(self, value):
self.stat.state = value
self.log.debug(self.state)
async def warm_up(self):
pass
async def input(self):
yield
raise NotImplementedError()
async def process(self, item):
yield
raise NotImplementedError()
async def update(self, result):
raise NotImplementedError()
async def shutdown(self):
pass
async def __call__(self):
self.state = "🔥 Warming up"
await self.warm_up()
self.stat._start_time = time()
try:
await self.main_cycle()
except Exception:
self.log.exception("MAIN CYCLE")
if not self.MUTE_EXCEPTION:
raise
self.stat.finished_time = time()
self.state = "🛑 Shutdown"
await self.shutdown()
self.state = "🏁 Finished"
async def main_cycle(self):
self.state = "⌛️ Ready to start"
await asyncio.gather(self._input_cycle(), self._result_cycle())
async def _result_cycle(self):
while True:
try:
result = await asyncio.wait_for(self.task_manager.take(), 1)
except asyncio.TimeoutError:
continue
if isinstance(result, TasksManager.Finish):
break
await self.update(result)
self.stat.updated_items += 1
async def _input_cycle(self):
self.stat.retries = 0
while not self.need_stop:
self.state = "🔎 Wait for new item"
async for item in self.input():
self.stat.input_items += 1
await self.task_manager.put(self._run_task(TaskInfo(item)))
self.stat.retries = None
if self.INPUT_RETRIES == 0:
# Need to run only one time
self.need_stop = True
continue
if self.stat.retries is None:
# Item found
self.stat.retries = 0
await asyncio.sleep(0)
continue
if self.stat.retries >= self.INPUT_RETRIES:
self.log.warning("Too many retries, i'm done",
retries=self.stat.retries)
self.need_stop = True
continue
# Retry logic
self.stat.retries += 1
self.state = f"🔎 Wait items, repeat №{self.stat.retries}"
await asyncio.sleep(self.stat.retries * self.WAIT_COEF)
await self.task_manager.stop()
async def _run_task(self, info: TaskInfo):
self.tasks.append(info)
info.update("🎬 Task started")
info.update(f"🛠 Processing")
async for result in self.process(info.item):
self.stat.returned_items += 1
info.update(f"🛠 {repr(result)}")
yield result
info.update(f"🛠 Processing")
self.stat.processed_items += 1
info.update("✅ Finish processing")
if info.processed_callback:
info.update("🤙 Run callback")
self.log.info("Run processed callback",
processed_callback=info.processed_callback)
await info.processed_callback
self.stat.finished_items += 1
info.update("🏁 Task finished")
self.tasks.remove(info)
async def take_error(self):
return await self.task_manager.take_error()
def __client_peer_disconnect(self, connection_name, hostname, port,
client_channel, client_obj):
Log.info('Disconnected to %s (%s:%d)' % (
connection_name, hostname, port
))
def init_all():
"""Init all modules."""
__index_modules()
initialized = [init(module_name) for module_name in __modules_data.copy()]
Log.info('%d modules initialized' % sum(initialized))
