def index() -> Results:
from my.facebook import events
from my.facebook import (
Post,
Conversation,
)
for e in events():
if isinstance(e, Exception):
yield e
continue
elif isinstance(e, Post):
for url in iter_urls(e.content):
yield Visit(
url=url,
dt=e.dt,
locator=Loc(title=e.action or ""),
context=e.content,
)
elif isinstance(e, Conversation):
for msg in e.messages:
for url in iter_urls(msg.content):
yield Visit(
url=url,
dt=msg.dt,
context=msg.content,
locator=Loc(title=msg.author),
)
else:
# ignore other events
continue
def index(*, render_markdown: bool = False) -> Results:
from my.discord import messages
# TODO: optionally import? this would break if someone
# hasnt installed promnesia like pip3 install '.[all]' to
# to install the markdown module for promnesia
from promnesia.sources.markdown import TextParser
for m in messages():
# hmm - extract URLs from attachments?
# Probably not very useful unless I extract info from them with url_metadata or something
context: str = m.content
# if render_markdown flag is enabled, render the text as markdown (HTML)
if render_markdown:
context = TextParser(m.content)._doc_ashtml()
# permalink back to this discord message
loc = Loc.make(title=m.channel.description, href=m.link)
for u in iter_urls(m.content):
yield Visit(
url=u,
dt=m.timestamp,
context=context,
locator=loc,
)
def index() -> Results:
from . import hpi
from my.hackernews import dogsheep
for item in dogsheep.items():
if isinstance(item, Exception):
yield item
continue
hn_url = item.permalink
title = "hackernews"
if item.title:
title = item.title
elif item.text_html:
title = item.text_html
title = textwrap.shorten(
title, width=79, placeholder="…",
break_long_words=True)
# The locator is always the HN story. If the story is a link (as
# opposed to a text post), we insert a visit such that the link
# will point back to the corresponding HN story.
loc = Loc.make(title=title, href=hn_url)
urls = [hn_url]
if item.url is not None:
urls.append(item.url)
for url in urls:
yield Visit(
url=url,
dt=item.created,
locator=loc,
context=title,
)
def index() -> Results:
from my.newsboat import events
for e in events():
# if I added this as a subscription
if e.added:
yield Visit(url=e.url,
dt=e.dt,
locator=Loc(title=f"Added {e.url}", href=e.url))
def index() -> Results:
from my.google import events
from my.google.models import (
HtmlComment,
HtmlEvent,
LikedVideo,
AppInstall,
Location,
)
for e in events():
if isinstance(e, Location):
continue
elif isinstance(e, AppInstall):
continue
elif isinstance(e, LikedVideo):
yield Visit(
url=e.link,
dt=e.dt,
context=e.desc,
locator=Loc(title=e.title, href=e.link),
)
elif isinstance(e, HtmlComment):
for url in e.links:
# todo: use url_metadata to improve locator?
# or maybe just extract first sentence?
yield Visit(
url=url,
dt=e.dt,
context=e.desc,
locator=Loc(title=e.desc, href=url),
)
elif isinstance(e, HtmlEvent):
# TODO: regex out title and use it as locator title?
for url in filter(lambda u: "youtube.com/channel" not in u, e.links):
yield Visit(
url=url,
dt=e.dt,
context=e.desc,
locator=Loc(title=e.desc, href=url),
)
else:
yield RuntimeError(f"Unhandled visit: {repr(e)}")
def err_ex():
for i in range(3):
if i == 1:
yield ExtractionError()
else:
yield Visit(
url=f'http://test{i}',
dt=datetime.utcfromtimestamp(0),
locator=Loc.make('whatever'),
)
def index() -> Results:
from my.ttt import history
for e in history():
for u in iter_urls(e.command):
yield Visit(
url=u,
dt=e.dt,
context=e.command,
locator=Loc(title=e.command, href=u),
)
def index() -> Results:
from my.old_forums import history
for p in history():
desc = f"{p.forum_name} - {p.post_title}"
loc = Loc(title=desc, href=p.post_url)
# visit directly to this link
yield Visit(url=p.post_url,
dt=p.dt,
locator=loc,
context=p.post_contents)
# visit to any links I mentioned in the contents
for url in iter_urls(p.post_contents):
yield Visit(
url=url,
dt=p.dt,
locator=loc,
context=p.post_contents,
)
def index() -> Results:
from my.grouvee import games
from grouvee_export.dal import Game
for g in games():
game_url = g.url
loc = Loc.make(title=f"Grouvee {g.name}", href=game_url)
for s in g.shelves:
yield Visit(
url=game_url, dt=s.added, locator=loc, context=f"{g.name} ({s.name})"
)
def index() -> Results:
from my.twitch.gdpr import events
for e in events():
if e.event_type == "pageview":
url = str(e.context)
yield Visit(
url=url,
dt=e.dt,
locator=Loc.make(title=f"Twitch {e.context}", href=url),
)
def index() -> Results:
from my.mpv import history
for play in history():
if play.is_stream:
url = play.path
yield Visit(
url=url,
dt=play.start_time,
duration=int(play.listen_time),
locator=Loc(title=play.media_title or play.path, href=url),
)
def index() -> Results:
from my.coding.commits import commits
for c in commits():
for url in iter_urls(c.message):
desc = f"{c.repo}\n{c.message}"
yield Visit(
url=url,
dt=c.committed_dt,
context=desc,
locator=Loc(title=f"{c.repo} {c.sha}", href=c.repo),
)
def index() -> Results:
from my.albums import history
for a in history():
if a.discogs_url is not None and a.listened_on is not None:
dt = datetime.combine(a.listened_on, datetime.min.time())
title = f"{a.album_name} - {a.cover_artists}"
# TODO: add stringified description of album as context?
yield Visit(
url=a.discogs_url,
dt=dt,
locator=Loc(title=title, href=a.discogs_url),
)
def index() -> Results:
from my.browsing import history
# TODO: expose a better locator from browserexport?
for v in history():
desc = None
if v.metadata is not None:
desc = v.metadata.title
yield Visit(
url=v.url,
dt=v.dt,
locator=Loc(title=desc or v.url, href=v.url),
)
def index() -> Results:
from my.ipython import history
for e in history():
if isinstance(e, Exception):
logger.exception(e)
continue
for u in iter_urls(e.command):
yield Visit(
url=u,
dt=e.dt,
context=e.command,
locator=Loc(title=e.command, href=u),
)
def index() -> Results:
from my.todotxt import events
emitted: Set[Tuple[str, str]] = set()
for e in events():
for u in iter_urls(e.todo.text):
key = (e.todo.text, u)
if key in emitted:
continue
yield Visit(
url=u,
dt=e.dt,
context=e.todo.text,
locator=Loc(title=e.todo.text, href=u),
)
emitted.add(key)
def index() -> Results:
from . import hpi
from my.smscalls import messages
for m in messages():
urls = extract_urls(m.message)
if len(urls) == 0:
continue
loc = Loc(title=f"SMS with {m.who} ({m.phone_number})")
for u in urls:
yield Visit(
url=u,
dt=m.dt,
context=m.message,
locator=loc,
)
def index() -> Results:
from my.trakt import history
from traktexport.dal import Movie, Episode
for e in history():
# TODO: filter/fix datetimes from before I created my account?
# TODO: use SiteIds to expand what this connects to (can link to imdb/moviedb?)
trakt_url = e.media_data.url
desc: str
if isinstance(e.media_data, Movie):
desc = e.media_data.title
elif isinstance(e.media_data, Episode):
# episode
desc = f"{e.media_data.show.title} - S{e.media_data.season}E{e.media_data.episode} - {e.media_data.title}"
else:
yield RuntimeError(f"Unhandled event {repr(e)}")
continue
yield Visit(
url=trakt_url,
dt=e.watched_at,
locator=Loc(title=e.media_data.title, href=trakt_url),
context=desc,
)