def parse_ompl(path: str) -> ElementTree.ElementTree:
"""Parse the OPML file and error out if it's in the wrong format"""
with open(path) as f:
try:
return ElementTree.parse(f)
except ElementTree.ParseError:
logger.error(f"Could not parse `{path}`.\nIt's probably not a proper OPML file.")
sys.exit(1)
def validate_config_overwrite(path: str, force: bool) -> None:
"""Make sure the --force flag is passed where applicable"""
if os.path.exists(path):
if force:
logger.warning(f"Overwriting existing configuration file in `{path}`...")
else:
logger.error(f"A file already exists in `{path}`.\nPass the --force flag to overwrite it.")
sys.exit(1)
def get_local_feed(feed_title: str) -> dict:
"""Get the feed if it exists in the feeds dict, or exit with an error"""
feed = config.feeds.get(feed_title)
if not feed:
logger.error(
f"Tried to fetch articles from an unknown feed `{feed_title}`")
sys.exit(1)
return feed
def parse(self, url: str) -> dict:
"""
Parse a single URL with the Mercury Parser and return the result
"""
result = self.get_parsed_doc(url)
if result.get("error"):
error_msg = result.get("message", "Unknown")
logger.error(f"Failed to parse {url}\nError: {error_msg}")
return {}
return result
def send_email_message(server: smtplib.SMTP, msg: EmailMessage) -> bool:
"""Send a single EmailMessage"""
logger.debug("Sending the email...")
try:
server.send_message(msg)
return True
except smtplib.SMTPException as e:
logger.error(
f"Caught an exception while trying to send an email.\nError: {e}")
return False
def validate_existing_feeds(title: str, force: bool) -> None:
"""Error out if no force flag was passed and the feed already exists"""
if title in config.feeds:
if force:
logger.confirm(f"Going to overwrite the following feed: {title}")
else:
logger.error(
f"The following feed already exists: {title}\nPass the --force flag if you'd like to overwrite it"
)
sys.exit(1)
def validate_parser() -> None:
"""Run various validations for the different parsers"""
parser = Parser(config.parser)
if parser == Parser.MERCURY:
try:
import docker # noqa
except ModuleNotFoundError:
logger.error(
"The `docker` module is not installed, but is required to use the `mercury` parser\n"
"Consider either switching to a different parser (by running `r2k config set -k parser --force`)\n"
"Or install the optional `docker` library by running `pip install 'r2k[docker]'`"
)
sys.exit(1)
def validate_conflicts(feeds: dict, force: bool) -> None:
"""Error out if the force flag was not passed and there are conflicts between new and existing feeds"""
new_feeds = set(feeds.keys())
old_feeds = set(config.feeds.keys())
conflicts = new_feeds & old_feeds
if conflicts:
conflicts_str = yaml.safe_dump(sorted(list(conflicts))) # Just here to like nicer in the output
if force:
logger.confirm(f"Going to overwrite the following feeds:\n{conflicts_str}")
else:
logger.error(
f"Found the following existing feeds:\n{conflicts_str}\n"
f"Pass the --force flag if you'd like to overwrite them."
)
sys.exit(1)
def config_set(key: str, force: bool) -> None:
"""Set a value in the config."""
if getattr(_config, key):
if force:
logger.warning("Overriding an existing value...")
else:
logger.error(
f"A value already exists for `{key}`.\nPass the --force flag to overwrite it."
)
sys.exit(1)
value = Prompt.get(key)
setattr(_config, key, value)
logger.info("Configuration successfully updated!")
def send_email_messages(msgs: List[EmailMessage]) -> int:
"""Send an email"""
messages_sent = 0
try:
logger.debug("Connecting to SMTP...")
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as server:
server.ehlo()
logger.debug("Logging into the SMTP server...")
server.login(config.send_from, config.password)
for msg in msgs:
if send_email_message(server, msg):
messages_sent += 1
logger.debug("Email sent successfully!")
except smtplib.SMTPException as e:
logger.error(
f"Caught an exception while trying to send an email.\nError: {e}")
return messages_sent
def feed_add(title: str, url: str, force: bool) -> None:
"""Add an RSS feed."""
validate_existing_feeds(title, force)
feeds = get_feeds_from_url(url)
if not feeds:
logger.error("Could not find an RSS feed")
sys.exit(1)
elif len(feeds) == 1:
feed = feeds[0]
else:
feed, _ = pick(feeds, "Please choose the correct feed from this list:")
config.feeds[title] = {"url": feed}
config.save()
logger.info("Successfully added the feed!")
def send_updates(unread_articles: List[Article], feed_title: str) -> None:
"""Iterate over `unread_articles`, and send each one to the kindle"""
if unread_articles:
if Parser(config.parser) == Parser.PUSH_TO_KINDLE:
successful_count = send_urls([(article.title, article.link)
for article in unread_articles])
else:
successful_count = send_epub_books(unread_articles, feed_title)
if successful_count:
logger.notice(
f"Successfully sent {successful_count} articles from the `{feed_title}` feed!"
)
else:
logger.error(
f"Failed to send any articles to `{feed_title}`. See errors above"
)
else:
logger.info(f"No new content for `{feed_title}`")
def __exit__(self, exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType]) -> bool:
"""
Context manager __exit__ for MercuryParser
1. Remove the container
2. Stop the program for any expected errors
"""
self.remove_container()
if exc_val:
if isinstance(exc_val, (DockerAPIError, ConnectionError)):
logger.error(
"Could not connect to Docker. Run with -v to get more details"
)
logger.debug(f"Error info:\n{exc_val}")
sys.exit(1)
else:
raise exc_val
return True
class MercuryParser(ParserBase):
"""
Represents the gateway to the Mercury Parser API
Relies on https://hub.docker.com/r/wangqiru/mercury-parser-api
"""
def __init__(self) -> None:
"""Constructor"""
self.container: Optional[Container] = None
self.client: docker.DockerClient = docker.from_env()
def run_mercury_container(self) -> Container:
"""Launch a new mercury-parser docker container"""
logger.debug("Launching a new mercury-parser Docker container...")
self.container = self.client.containers.run(
"wangqiru/mercury-parser-api:latest",
detach=True,
ports={f"{MERCURY_PORT}/tcp": MERCURY_PORT},
name=CONTAINER_NAME,
)
return self.container
def clean_existing_containers(self) -> None:
"""Remove any existing mercury parser API containers"""
all_containers = self.client.containers.list(all=True, sparse=True)
for container in all_containers:
if container.attrs["Names"] == [f"/{CONTAINER_NAME}"]:
logger.debug(
"Found an existing container with the same name...")
self.remove_container(container)
def remove_container(self, container: Optional[Container] = None) -> None:
"""Stop and remove a docker container"""
container = container or self.container
if container:
logger.debug("Stopping container...")
container.stop()
logger.debug("Removing container...")
container.remove()
def __enter__(self) -> MercuryParser:
"""
Context manager __enter__ for MercuryParser
1. Clean any existing containers
2. Spin a local, dockerized version of the mercury parser API
3. Validate it's up
4. Return the MercuryParser instance
"""
self.clean_existing_containers()
self.run_mercury_container()
self.validate_container_is_up()
return self
def __exit__(self, exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType]) -> bool:
"""
Context manager __exit__ for MercuryParser
1. Remove the container
2. Stop the program for any expected errors
"""
self.remove_container()
if exc_val:
if isinstance(exc_val, (DockerAPIError, ConnectionError)):
logger.error(
"Could not connect to Docker. Run with -v to get more details"
)
logger.debug(f"Error info:\n{exc_val}")
sys.exit(1)
else:
raise exc_val
return True
@staticmethod
def validate_container_is_up() -> None:
"""Try to connect to the mercury parser service several times. Quit app if not successful"""
errors = set()
logger.debug(
f"Launched container at {BASE_MERCURY_URL}. Validating it's up...")
while retries := CONNECTION_ATTEMPTS:
try:
requests.get(BASE_MERCURY_URL)
logger.debug("Connected!")
return
except ConnectionError as e:
errors.add(e)
sleep(1)
retries -= 1
logger.error(
"Could not connect to the mercury-parser Docker container")
if errors:
errors_str = "\n".join(str(error) for error in errors)
logger.error(f"Error info:{errors_str}")
sys.exit(1)