async def update_specific_config(self, filename: str): """Reads the provided config file and updates the interested monitors/scrapers""" self.general_logger.debug(f"File {filename} has changed!") try: with open(filename, "r") as f: j = json.load(f) except JSONDecodeError: self.general_logger.warning( f"File {filename} has changed but contains invalid json data") return splits = filename.split(os.path.sep) commands = [] # List[Cmd] sock_paths = [] # type: List[str] # if it's from the monitors folder: if "monitors" in filename.split(os.path.sep): sockets = self.monitor_sockets elif "scrapers" in filename.split(os.path.sep): sockets = self.scraper_sockets else: self.general_logger.debug("File not useful.") return # we are interested in configs, whitelists, blacklists, webhooks if splits[-1] == "whitelists.json": cmd = COMMANDS.SET_SPECIFIC_WHITELIST elif splits[-1] == "configs.json": cmd = COMMANDS.SET_SPECIFIC_CONFIG elif splits[-1] == "blacklists.json": cmd = COMMANDS.SET_SPECIFIC_BLACKLIST elif splits[-1] == "webhooks.json": cmd = COMMANDS.SET_SPECIFIC_WEBHOOKS else: return # for every monitor socket for name in sockets: if name in j: sock_path = sockets[name] c = Cmd() c.cmd = cmd # send only the corresponding part to the monitor c.payload = j[name] commands.append(c) sock_paths.append(sock_path) # prepare to make all the async requests tasks = [] for sock_path, command in zip(sock_paths, commands): tasks.append(self.make_request(sock_path, command)) # send the requests responses = await asyncio.gather(*tasks) # List[Response] for response in responses: if response.error.value: self.general_logger.warning( f"Failed to update config: {response.error}")
async def delete(self): cmd = Cmd() cmd.cmd = COMMANDS.MM_STOP_MONITOR_SCRAPER cmd.payload = json.loads(self.request.body) r = await send_to_moman(cmd) if not r.error.value: self.set_status(400, r.reason) self.write(r.get_json())
async def post(self): cmd = Cmd() cmd.cmd = COMMANDS.MM_ADD_MONITOR cmd.payload = json.loads(self.request.body) r = await send_to_moman(cmd) if r.error.value: self.set_status(400, r.reason) self.write(r.get_json())
async def on_server_stop(self): """ Routine that runs on server stop. Shuts down the monitor manager """ async with self._loop_lock: # stop the config watcher self.watcher.stop() self.watcher.join() if self.shutdown_all_on_exit: # get all the existing sockets sockets = [] # type: List[str] tasks = [] for sockname in os.listdir( self.config["GlobalConfig"]["socket_path"]): if sockname.startswith("Scraper.") or sockname.startswith( "Monitor."): cmd = Cmd() cmd.cmd = COMMANDS.STOP sockets.append(sockname) self.general_logger.info(f"Stopping {sockname}...") tasks.append( self.make_request( f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname}", cmd, )) # send request to stop responses = await asyncio.gather(*tasks ) # type: List[Response] for sockname, r in zip(sockets, responses): # if an error happened... if r.error.value: # if the socket was not used remove it if r.error == ERRORS.SOCKET_COULDNT_CONNECT: os.remove( os.path.sep.join([ self.config["GlobalConfig"]["socket_path"], sockname, ])) self.general_logger.info( f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname} was removed because unavailable" ) # else something else happened, dont do anything else: self.general_logger.warning( f"Error occurred while attempting to stop {sockname}: {r.error}" ) # ok else: self.general_logger.info( f"{sockname} was successfully stopped") self._asyncio_loop.stop() self.general_logger.info("Shutting down...") return okResponse()
async def on_stop_scraper(self, cmd: Cmd) -> Response: r = badResponse() success, missing = cmd.has_valid_args(self.stop_args) if success: payload = cast(Dict[str, Any], cmd.payload) socket = f"{self.config['GlobalConfig']['socket_path']}/Scraper.{payload['name']}" command = Cmd() command.cmd = COMMANDS.STOP self.general_logger.debug(f"Sending STOP to {socket}...") r = await self.make_request(socket, command) self.general_logger.debug(f"Sent STOP to {socket}") else: r.error = ERRORS.MISSING_PAYLOAD_ARGS r.info = f"Missing arguments: {missing}" return r
async def on_get_monitor_shoes(self, cmd: Cmd) -> Response: success, missing = cmd.has_valid_args(self.getter_args) if success: payload = cast(Dict[str, Any], cmd.payload) c = Cmd() c.cmd = COMMANDS.SET_SHOES r = await self.make_request( f"{self.config['GlobalConfig']['socket_path']}/Monitor.{payload['name']}", c, ) return r else: r = badResponse() r.error = ERRORS.MISSING_PAYLOAD_ARGS r.info = f"{missing}" return r
async def specific_config_getter(self, cmd: Cmd, command: COMMANDS, is_monitor: bool): success, missing = cmd.has_valid_args(self.getter_args) if success: payload = cast(Dict[str, Any], cmd.payload) c = Cmd() c.cmd = command r = await self.make_request( f"{self.config['GlobalConfig']['socket_path']}/{'Monitor' if is_monitor else 'Scraper'}.{payload['name']}", c, ) return r else: r = badResponse() r.error = ERRORS.MISSING_PAYLOAD_ARGS r.info = f"{missing}" return r
async def get_alive_sockets(self, sockets: List[str]) -> List[str]: """ Ping the provided sockets and return a list of alive sockets """ tasks = [] for socket in sockets: cmd = Cmd() cmd.cmd = COMMANDS.PING tasks.append(self.make_request(socket, cmd)) responses = await asyncio.gather(*tasks) # type: List[Response] alive = [] for response, socket in zip(responses, sockets): if not response.error.value: alive.append(socket) return alive
try: COMMANDS[key] print(key) except: pass exit(0) cmd = COMMANDS.__dict__.get(args[1], None) if cmd is None: try: cmd = int(args[1]) except: print("cmd is not a valid COMMANDS nor an int.") exit(1) string_cmd = args[1] payload = {} if len(args) > 2: if not len(args) % 2: for index, term in enumerate(args[2:]): if not index % 2: if not term.startswith("--"): print('You must start every payload key with "--"') exit(4) payload[term[2:]] = args[3 + index] else: print("Incorrect number of payload options!") exit(3) command = Cmd() command.cmd = cmd command.payload = payload send(command)
async def get(self): cmd = Cmd() cmd.cmd = COMMANDS.MM_GET_MONITOR_STATUS r = await send_to_moman(cmd) self.write(r.get_json())
from kekmonitors.comms.msg import Cmd from kekmonitors.config import COMMANDS from kekmonitors.monitor_manager_cli import send if __name__ == "__main__": cmd = Cmd() cmd.cmd = COMMANDS.MM_STOP_MONITOR_MANAGER send(cmd)