async def on_server_stop(self):
"""
Routine that runs on server stop. Shuts down the monitor manager
"""
async with self._loop_lock:
# stop the config watcher
self.watcher.stop()
self.watcher.join()
if self.shutdown_all_on_exit:
# get all the existing sockets
sockets = [] # type: List[str]
tasks = []
for sockname in os.listdir(
self.config["GlobalConfig"]["socket_path"]):
if sockname.startswith("Scraper.") or sockname.startswith(
"Monitor."):
cmd = Cmd()
cmd.cmd = COMMANDS.STOP
sockets.append(sockname)
self.general_logger.info(f"Stopping {sockname}...")
tasks.append(
self.make_request(
f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname}",
cmd,
))
# send request to stop
responses = await asyncio.gather(*tasks
) # type: List[Response]
for sockname, r in zip(sockets, responses):
# if an error happened...
if r.error.value:
# if the socket was not used remove it
if r.error == ERRORS.SOCKET_COULDNT_CONNECT:
os.remove(
os.path.sep.join([
self.config["GlobalConfig"]["socket_path"],
sockname,
]))
self.general_logger.info(
f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname} was removed because unavailable"
)
# else something else happened, dont do anything
else:
self.general_logger.warning(
f"Error occurred while attempting to stop {sockname}: {r.error}"
)
# ok
else:
self.general_logger.info(
f"{sockname} was successfully stopped")
self._asyncio_loop.stop()
self.general_logger.info("Shutting down...")
return okResponse()
async def update_common_config(self, filename: str):
"""Reads the provided config file and updates the interested monitors/scrapers"""
self.general_logger.debug(f"File {filename} has changed!")
try:
with open(filename, "r") as f:
j = json.load(f)
except JSONDecodeError:
self.general_logger.warning(
f"File {filename} has changed but contains invalid json data")
return
splits = filename.split(os.path.sep)
for sockets in (self.monitor_sockets, self.scraper_sockets):
commands = [] # List[Cmd]
sock_paths = [] # type: List[str]
# we are interested in configs, whitelists, blacklists, webhooks
if splits[-1] == "whitelists.json":
cmd = COMMANDS.SET_COMMON_WHITELIST
elif splits[-1] == "configs.json":
cmd = COMMANDS.SET_COMMON_CONFIG
elif splits[-1] == "blacklists.json":
cmd = COMMANDS.SET_COMMON_BLACKLIST
elif splits[-1] == "webhooks.json":
cmd = COMMANDS.SET_COMMON_WEBHOOKS
else:
return
# for every monitor socket
for name in sockets:
if name in j:
sock_path = sockets[name]
c = Cmd()
c.cmd = cmd
# send only the corresponding part to the monitor
c.payload = j[name]
commands.append(c)
sock_paths.append(sock_path)
# prepare to make all the async requests
tasks = []
for sock_path, command in zip(sock_paths, commands):
tasks.append(self.make_request(sock_path, command))
# send the requests
responses = await asyncio.gather(*tasks) # List[Response]
for response in responses:
if response.error.value:
self.general_logger.warning(
f"Failed to update config: {response.error}")
async def on_stop_scraper(self, cmd: Cmd) -> Response:
r = badResponse()
success, missing = cmd.has_valid_args(self.stop_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
socket = f"{self.config['GlobalConfig']['socket_path']}/Scraper.{payload['name']}"
command = Cmd()
command.cmd = COMMANDS.STOP
self.general_logger.debug(f"Sending STOP to {socket}...")
r = await self.make_request(socket, command)
self.general_logger.debug(f"Sent STOP to {socket}")
else:
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"Missing arguments: {missing}"
return r
async def make_request(self,
socket_path: str,
cmd: Cmd,
expect_response: bool = True) -> Response:
if os.path.exists(socket_path):
try:
reader, writer = await asyncio.open_unix_connection(socket_path
)
writer.write(cmd.get_bytes())
writer.write_eof()
if expect_response:
r = Response(await reader.read())
writer.close()
return r
return okResponse()
except ConnectionRefusedError:
self.server_logger.exception(
f"Couldn't connect to socket {socket_path}")
r = badResponse()
r.error = ERRORS.SOCKET_COULDNT_CONNECT
return r
r = badResponse()
r.error = ERRORS.SOCKET_DOESNT_EXIST
return r
async def on_get_monitor_shoes(self, cmd: Cmd) -> Response:
success, missing = cmd.has_valid_args(self.getter_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
c = Cmd()
c.cmd = COMMANDS.SET_SHOES
r = await self.make_request(
f"{self.config['GlobalConfig']['socket_path']}/Monitor.{payload['name']}",
c,
)
return r
else:
r = badResponse()
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"{missing}"
return r
async def specific_config_setter(self, cmd: Cmd, filename: str,
is_monitor: bool):
success, missing = cmd.has_valid_args(self.setter_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
cp = os.path.sep.join((
self.config["GlobalConfig"]["config_path"],
"monitors" if is_monitor else "scrapers",
filename,
))
with open(
cp,
"r",
) as rf:
f = json.load(rf)
f[payload["name"]] = payload["payload"]
with open(
cp,
"w",
) as wf:
json.dump(f, wf)
else:
r = badResponse()
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"{missing}"
return r
async def specific_config_getter(self, cmd: Cmd, command: COMMANDS,
is_monitor: bool):
success, missing = cmd.has_valid_args(self.getter_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
c = Cmd()
c.cmd = command
r = await self.make_request(
f"{self.config['GlobalConfig']['socket_path']}/{'Monitor' if is_monitor else 'Scraper'}.{payload['name']}",
c,
)
return r
else:
r = badResponse()
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"{missing}"
return r
async def get_alive_sockets(self, sockets: List[str]) -> List[str]:
"""
Ping the provided sockets and return a list of alive sockets
"""
tasks = []
for socket in sockets:
cmd = Cmd()
cmd.cmd = COMMANDS.PING
tasks.append(self.make_request(socket, cmd))
responses = await asyncio.gather(*tasks) # type: List[Response]
alive = []
for response, socket in zip(responses, sockets):
if not response.error.value:
alive.append(socket)
return alive
async def on_stop_monitor_scraper(self, cmd: Cmd) -> Response:
r = badResponse()
success, missing = cmd.has_valid_args(self.stop_args)
if success:
r1, r2 = await asyncio.gather(self.on_stop_monitor(cmd),
self.on_stop_scraper(cmd))
r.error = (ERRORS.OK if not r1.error.value and not r2.error.value
else ERRORS.MM_COULDNT_STOP_MONITOR_SCRAPER)
r.info = f"Monitor: {r1.error.name}, Scraper: {r2.error.name}"
if r.error.value and r.error:
self.general_logger.warning(
f"Couldn't stop monitor and scraper")
kekmonitors.utils.tools.dump_error(self.general_logger, r)
else:
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"Missing arguments: {missing}"
return r
async def on_add_monitor(self, cmd: Cmd) -> Response:
r = badResponse()
success, missing = cmd.has_valid_args(self.add_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
db_monitor = self.register_db["monitors"].find_one(
{"name": payload["name"]})
if db_monitor:
success, reason = await self.add_monitor(
db_monitor["path"], payload)
if success:
r = okResponse()
else:
r.error = ERRORS.MM_COULDNT_ADD_MONITOR
r.info = reason
else:
r.error = ERRORS.MONITOR_NOT_REGISTERED
r.info = f"Tried to add monitor {payload['name']} but it was not found in the db. Did you start it at least once manually?"
else:
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"Missing arguments: {missing}"
return r
async def make_request(socket_path: str,
cmd: Cmd,
expect_response=True) -> Response:
"""
Send `cmd` to `socket_path` and return the response if `expect_response` is True, else `okResponse()`.
"""
if os.path.exists(socket_path):
try:
reader, writer = await asyncio.open_unix_connection(socket_path)
writer.write(cmd.get_bytes())
writer.write_eof()
if expect_response:
response = Response(await reader.read())
writer.close()
return response
return okResponse()
except ConnectionRefusedError:
pass
r = badResponse()
r.error = ERRORS.SOCKET_DOESNT_EXIST
return r
async def _handle_msg(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
"""Handle incoming messages."""
msg = Cmd(await reader.read())
addr = writer.get_extra_info("peername")
print_addr = addr if addr else "localhost"
self.server_logger.debug(f"Received from {print_addr}")
if msg.cmd in self.cmd_to_callback:
self.server_logger.debug(f"Got cmd: {msg.cmd}")
response = await self.cmd_to_callback[msg.cmd](msg)
else:
self.server_logger.warning(
f"Got unrecognized command: {msg.cmd.value}")
response = badResponse()
response.error = ERRORS.UNRECOGNIZED_COMMAND
writer.write(response.get_bytes())
writer.write_eof()
await writer.drain()
self.server_logger.debug(f"Closed connection from {print_addr}")
writer.close()
try:
COMMANDS[key]
print(key)
except:
pass
exit(0)
cmd = COMMANDS.__dict__.get(args[1], None)
if cmd is None:
try:
cmd = int(args[1])
except:
print("cmd is not a valid COMMANDS nor an int.")
exit(1)
string_cmd = args[1]
payload = {}
if len(args) > 2:
if not len(args) % 2:
for index, term in enumerate(args[2:]):
if not index % 2:
if not term.startswith("--"):
print('You must start every payload key with "--"')
exit(4)
payload[term[2:]] = args[3 + index]
else:
print("Incorrect number of payload options!")
exit(3)
command = Cmd()
command.cmd = cmd
command.payload = payload
send(command)
async def get(self):
cmd = Cmd()
cmd.cmd = COMMANDS.MM_GET_MONITOR_STATUS
r = await send_to_moman(cmd)
self.write(r.get_json())
from kekmonitors.comms.msg import Cmd
from kekmonitors.config import COMMANDS
from kekmonitors.monitor_manager_cli import send
if __name__ == "__main__":
cmd = Cmd()
cmd.cmd = COMMANDS.MM_STOP_MONITOR_MANAGER
send(cmd)