async def make_request(self,
socket_path: str,
cmd: Cmd,
expect_response: bool = True) -> Response:
if os.path.exists(socket_path):
try:
reader, writer = await asyncio.open_unix_connection(socket_path
)
writer.write(cmd.get_bytes())
writer.write_eof()
if expect_response:
r = Response(await reader.read())
writer.close()
return r
return okResponse()
except ConnectionRefusedError:
self.server_logger.exception(
f"Couldn't connect to socket {socket_path}")
r = badResponse()
r.error = ERRORS.SOCKET_COULDNT_CONNECT
return r
r = badResponse()
r.error = ERRORS.SOCKET_DOESNT_EXIST
return r
async def on_get_status(self, cmd: Cmd) -> Response:
ms = await self.on_get_monitor_status(cmd)
ss = await self.on_get_scraper_status(cmd)
response = okResponse()
msp = cast(Dict[str, Any], ms.payload) # type: Dict[str, Any]
ssp = cast(Dict[str, Any], ss.payload) # type: Dict[str, Any]
response.payload = {"monitors": msp, "scrapers": ssp}
return response
async def on_server_stop(self):
"""
Routine that runs on server stop. Shuts down the monitor manager
"""
async with self._loop_lock:
# stop the config watcher
self.watcher.stop()
self.watcher.join()
if self.shutdown_all_on_exit:
# get all the existing sockets
sockets = [] # type: List[str]
tasks = []
for sockname in os.listdir(
self.config["GlobalConfig"]["socket_path"]):
if sockname.startswith("Scraper.") or sockname.startswith(
"Monitor."):
cmd = Cmd()
cmd.cmd = COMMANDS.STOP
sockets.append(sockname)
self.general_logger.info(f"Stopping {sockname}...")
tasks.append(
self.make_request(
f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname}",
cmd,
))
# send request to stop
responses = await asyncio.gather(*tasks
) # type: List[Response]
for sockname, r in zip(sockets, responses):
# if an error happened...
if r.error.value:
# if the socket was not used remove it
if r.error == ERRORS.SOCKET_COULDNT_CONNECT:
os.remove(
os.path.sep.join([
self.config["GlobalConfig"]["socket_path"],
sockname,
]))
self.general_logger.info(
f"{self.config['GlobalConfig']['socket_path']}{os.path.sep}{sockname} was removed because unavailable"
)
# else something else happened, dont do anything
else:
self.general_logger.warning(
f"Error occurred while attempting to stop {sockname}: {r.error}"
)
# ok
else:
self.general_logger.info(
f"{sockname} was successfully stopped")
self._asyncio_loop.stop()
self.general_logger.info("Shutting down...")
return okResponse()
async def on_server_stop(self) -> Response:
self.general_logger.debug("Waiting for loop to complete...")
async with self._loop_lock:
pass
self.general_logger.debug("Loop is completed, starting shutdown...")
await self.on_async_shutdown()
self._asyncio_loop.stop()
self.general_logger.debug("Shutting down webhook manager...")
self.webhook_manager.quit()
self.on_shutdown()
return okResponse()
async def on_get_scraper_status(self, cmd: Cmd) -> Response:
process_status = {}
for class_name in self.scraper_processes:
start = self.scraper_processes[class_name]["start"].strftime(
"%m/%d/%Y, %H:%M:%S")
process_status[class_name] = {
"Started at": start,
"PID": self.scraper_processes[class_name]["process"].pid,
}
sockets_status = {}
# for class_name in self.scraper_sockets:
# sockets_status[class_name] = {class_name: self.scraper_sockets[class_name]}
sockets_status = self.scraper_sockets
response = okResponse()
response.payload = {
"monitored_processes": process_status,
"available_sockets": sockets_status,
}
return response
async def on_add_monitor(self, cmd: Cmd) -> Response:
r = badResponse()
success, missing = cmd.has_valid_args(self.add_args)
if success:
payload = cast(Dict[str, Any], cmd.payload)
db_monitor = self.register_db["monitors"].find_one(
{"name": payload["name"]})
if db_monitor:
success, reason = await self.add_monitor(
db_monitor["path"], payload)
if success:
r = okResponse()
else:
r.error = ERRORS.MM_COULDNT_ADD_MONITOR
r.info = reason
else:
r.error = ERRORS.MONITOR_NOT_REGISTERED
r.info = f"Tried to add monitor {payload['name']} but it was not found in the db. Did you start it at least once manually?"
else:
r.error = ERRORS.MISSING_PAYLOAD_ARGS
r.info = f"Missing arguments: {missing}"
return r
async def make_request(socket_path: str,
cmd: Cmd,
expect_response=True) -> Response:
"""
Send `cmd` to `socket_path` and return the response if `expect_response` is True, else `okResponse()`.
"""
if os.path.exists(socket_path):
try:
reader, writer = await asyncio.open_unix_connection(socket_path)
writer.write(cmd.get_bytes())
writer.write_eof()
if expect_response:
response = Response(await reader.read())
writer.close()
return response
return okResponse()
except ConnectionRefusedError:
pass
r = badResponse()
r.error = ERRORS.SOCKET_DOESNT_EXIST
return r
async def _on_ping(self, cmd: Cmd) -> Response:
return okResponse()
async def on_server_stop(self):
return okResponse()