def get(url):
if not config.get("force_fetch") and is_url_in_cache(url):
return read_cache(url)
contents = get_from_remote(url).text
write_cache(url, contents)
return contents
def generate_rr_chapters(self):
for chapter_url in self.story_index.chapter_urls:
print(f"Fetching chapter {chapter_url}")
chapter_html = request_dispatcher.get(chapter_url)
if config.get("comments") != "none":
comments_html = "" # TODO
else:
comments_html = ""
rr_chapter = RoyalroadChapter(str(chapter_html),
str(comments_html))
print("OK")
yield rr_chapter
def text_for_chapter(self, chapter):
if len(self.story.chapters) > 1:
chapter_header = f"<h2 class='chapter-header'><em>{chapter.title}</em></h2>"
else:
chapter_header = ""
if config.get("comments") != "none":
comments_text = (
"<h3>Comments</h3>"
+ "<div class='comments'>"
+ "".join([comment.as_html() for comment in chapter.comments])
+ "</div>"
)
else:
comments_text = ""
return chapter_header + chapter.text + comments_text
def generate_ao3_chapters(self):
for chapter_info in self.fetch_chapters_info():
chapter_url = chapter_info.url
print(f"Fetching chapter {chapter_url}")
chapter_html = request_dispatcher.get(chapter_url)
if config.get("comments") != "none":
comments_html = "" # TODO
else:
comments_html = ""
ao3_chapter = AO3Chapter(str(chapter_html),
str(comments_html),
date_published=chapter_info.date)
print("OK")
yield ao3_chapter
def __init__(self,
*,
title=None,
author=None,
summary=None,
chapters,
date_fetched=None):
self.title = title
self.author = author
self.summary = summary
self.chapters = chapters
if date_fetched is None:
self.date_fetched = datetime.now().timestamp()
else:
self.date_fetched = date_fetched
if config.get("comments") == "author":
for chapter in self.chapters:
chapter.prune_comments(self.author)
def generate_ffnet_chapters(self):
for n in itertools.count(1):
chapter_url = self.generate_chapter_url(n)
print(f"Fetching chapter {n} ({chapter_url})")
chapter_html = request_dispatcher.get(chapter_url)
if config.get("comments") != "none":
reviews_url = self.generate_chapter_reviews_url(n)
print(f"Fetching chapter {n} reviews ({reviews_url})")
reviews_html = request_dispatcher.get(reviews_url)
else:
reviews_html = ""
ffnet_chapter = FFNetChapter(str(chapter_html), str(reviews_html))
print("OK")
yield ffnet_chapter
if (ffnet_chapter.is_last_chapter()
or ffnet_chapter.is_single_chapter_story()):
print("Done")
return
def test_nonexistent_fetcher_override(self, mocker):
config.load(test_config, fetcher=mocker.MagicMock(__module__="rabbits"))
assert config.get("comments") == "all"
def test_load(self):
config.load(test_config, fetcher=None)
assert config.get("archive_location") == "/path/to/archive"
assert config.get("comments") == "all"
def setup_reddit():
return praw.Reddit(
client_id=config.get(["reddit", "app"]),
client_secret=config.get(["reddit", "secret"]),
user_agent=config.get("user_agent"),
)
def ensure_cache_dir_exists():
cache_path = Path(config.get("http_cache_location"))
if not cache_path.exists():
cache_path.mkdir(parents=True)
def get_from_remote(url):
"""Wait MIN_DELAY to 2*MIN_DELAY seconds, then perform an HTTP get request and return the response"""
delay = MIN_DELAY + random.random() * MIN_DELAY
time.sleep(delay)
return requests.get(url, headers={"User-Agent": config.get("user_agent")})
def repo_ready_for_write(self):
return (not git.repo_is_dirty() or not config.get("write_archive")
or config.get("clobber"))
def comments(self):
if config.get("comments") != "none":
return self.all_comments
return []
def repo_path():
return config.get("archive_location")
def output_filename(self):
return config.get("outfile") or os.path.join(
config.get("epub_location"), self.story.filename
)