async def collect_site(self) -> Dict[Path, Dict[str, Interaction]]:
"""
Return replies to posts.
This is done by scraping the capsule and searching for "Re: " posts.
"""
response = await self.client.get(URL(self.url))
payload = await response.read()
content = payload.decode("utf-8")
posts = get_posts(self.root, self.config)
interactions: Dict[Path, Dict[str, Interaction]] = defaultdict(dict)
for line in content.split("\n"):
if not line.startswith("=>"):
continue
_, url, name = re.split(r"\s+", line, 2)
for post in posts:
reply = f"Re: {post.title}"
if reply in name and await self._link_in_post(post.url, url):
id_ = f"reply,{url}"
interactions[post.path][id_] = Interaction(
id=id_,
name=name,
url=url,
type="reply",
)
return interactions
async def test_get_posts(fs: FakeFilesystem, root: Path,
config: Config) -> None:
"""
Test ``get_posts``.
"""
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_dir(root / "posts/one")
fs.create_file(root / "posts/one/index.mkd")
with freeze_time("2021-01-02T00:00:00Z"):
fs.create_dir(root / "posts/two")
fs.create_file(root / "posts/two/index.mkd")
posts = get_posts(root, config)
assert len(posts) == 2
assert posts[0].path == Path(root / "posts/two/index.mkd")
assert posts[1].path == Path(root / "posts/one/index.mkd")
# test limited number of posts returned
posts = get_posts(root, config, 1)
assert len(posts) == 1
async def process_site(self, force: bool = False) -> None:
"""
Process the entire site.
"""
posts = get_posts(self.root, self.config)
# build index and feed
for asset in self.site_templates:
path = self.root / "build" / self.path / asset
template_name = f"{self.template_base}{asset}"
self._build_index(path, template_name, posts, force)
# template for groups (tags and categories)
template_name = f"{self.template_base}group{self.extension}"
# group posts by tag
tags = defaultdict(list)
for post in posts:
for tag in sorted(post.tags):
tags[tag].append(post)
for tag, tag_posts in tags.items():
path = self.root / "build" / self.path / "tags" / (tag +
self.extension)
self._build_index(path, template_name, tag_posts, force, title=tag)
# group tags by categories
categories = defaultdict(list)
for post in posts:
for category in sorted(post.categories):
categories[category].append(post)
for category, category_posts in categories.items():
path = (self.root / "build" / self.path / "categories" /
(category + self.extension))
title = self.config.categories[category].label
subtitle = self.config.categories[category].description
self._build_index(
path,
template_name,
category_posts,
force,
title=title,
subtitle=subtitle,
)
async def run( # pylint: disable=too-many-locals
root: Path,
force: bool = False,
) -> None:
"""
Build blog from Markdown files and online interactions.
"""
_logger.info("Building blog")
config = get_config(root)
_logger.debug(config)
build = root / "build"
if not build.exists():
_logger.info("Creating `build/` directory")
build.mkdir()
posts = get_posts(root, config)
# collect interactions from posts/site
tasks = []
post_interactions: Dict[Path, Dict[str, Interaction]] = defaultdict(dict)
_logger.info("Collecting interactions from posts")
announcers = get_announcers(root, config, Scope.POST)
for post in posts:
for name, announcer in announcers.items():
if name in post.announcers:
task = asyncio.create_task(
collect_post(post, announcer, post_interactions), )
tasks.append(task)
_logger.info("Collecting interactions from site")
announcers = get_announcers(root, config, Scope.SITE)
for announcer in announcers.values():
task = asyncio.create_task(collect_site(announcer, post_interactions))
tasks.append(task)
await asyncio.gather(*tasks)
# store new interactions
tasks = []
for path, interactions in post_interactions.items():
task = asyncio.create_task(save_interactions(path.parent,
interactions))
tasks.append(task)
await asyncio.gather(*tasks)
# run assistants
tasks = []
_logger.info("Running post assistants")
assistants = get_assistants(root, config, Scope.POST)
for post in posts:
for assistant in assistants.values():
task = asyncio.create_task(assistant.process_post(post, force))
tasks.append(task)
_logger.info("Running site assistants")
assistants = get_assistants(root, config, Scope.SITE)
for assistant in assistants.values():
task = asyncio.create_task(assistant.process_site(force))
tasks.append(task)
await asyncio.gather(*tasks)
# build posts/site
tasks = []
builders = get_builders(root, config)
_logger.info("Processing posts")
for post in posts:
for builder in builders.values():
task = asyncio.create_task(builder.process_post(post, force))
tasks.append(task)
_logger.info("Processing site")
for builder in builders.values():
task = asyncio.create_task(builder.process_site(force))
tasks.append(task)
await asyncio.gather(*tasks)
async def run( # pylint: disable=too-many-locals
root: Path,
force: bool = False,
) -> None:
"""
Publish blog.
"""
_logger.info("Publishing blog")
config = get_config(root)
_logger.debug(config)
# publish site
publishings = load_yaml(root / PUBLISHINGS_FILENAME, Publishing)
tasks = []
for name, publisher in get_publishers(root, config).items():
since = publishings[name].timestamp if name in publishings else None
task = asyncio.create_task(
publish_site(name, publisher, publishings, since, force), )
tasks.append(task)
await asyncio.gather(*tasks)
# persist publishings
with open(root / PUBLISHINGS_FILENAME, "w", encoding="utf-8") as output:
yaml.dump(
{
name: publishing.dict()
for name, publishing in publishings.items()
},
output,
)
# announcements
tasks = []
# announce site
site_announcements = load_yaml(root / ANNOUNCEMENTS_FILENAME, Announcement)
last_published = max(publishing.timestamp
for publishing in publishings.values())
announcers = get_announcers(root, config, Scope.SITE)
for name, announcer in announcers.items():
if (name in site_announcements
and (site_announcements[name].timestamp +
timedelta(seconds=site_announcements[name].grace_seconds))
>= last_published):
# already announced after last published
_logger.info("Announcer %s is up-to-date", name)
continue
task = asyncio.create_task(
announce_site(name, announcer, site_announcements))
tasks.append(task)
# announce posts
modified_post_announcements: Dict[Path, Dict[str, Announcement]] = {}
announcers = get_announcers(root, config, Scope.POST)
for post in get_posts(root, config):
path = post.path.parent / ANNOUNCEMENTS_FILENAME
post_announcements = load_yaml(path, Announcement)
post_announcers = {
name: announcers[name]
for name in post.announcers
if name in announcers and name not in post_announcements
}
for name, announcer in post_announcers.items():
task = asyncio.create_task(
announce_post(name, announcer, post, post_announcements), )
tasks.append(task)
# store new announcements to persist later
if post_announcers:
modified_post_announcements[post.path] = post_announcements
await asyncio.gather(*tasks)
# persist new announcements
tasks = []
task = asyncio.create_task(save_announcements(root, site_announcements))
for post_path, announcements in modified_post_announcements.items():
task = asyncio.create_task(
save_announcements(post_path.parent, announcements))
tasks.append(task)
await asyncio.gather(*tasks)