def get_items(self):
yield node(
'Submissions',
children=[
node( # TODO can also be iterable?
dt_heading(s.created, link(title=s.title, url=s.url)),
body=s.text,
) for s in submissions()
])
yield node(
'Comments', # TODO parent thread??
children=[
node(
dt_heading(c.created, link(title=c.url, url=c.url)),
body=c.text,
) for c in comments()
],
)
yield node('Upvoted',
children=[
node(
dt_heading(u.created, link(title=u.title,
url=u.url)),
body=u.text,
) for u in upvoted()
])
def get_items(self) -> Mirror.Results:
for a in articles():
yield node(
heading=dt_heading(
a.added,
link(title='pocket', url=a.pocket_link) + ' · ' + link(title=a.title, url=a.url)
),
children=[node(
heading=dt_heading(hl.created, hl.text)
) for hl in a.highlights]
)
def get_items(self):
for page in get_pages():
yield node(heading=dt_heading(
page.dt, link(title=page.title, url=page.link)),
children=[
node(heading=dt_heading(
hl.dt, link(title='context', url=hl.hyp_link)),
tags=hl.tags,
body=hl.highlight,
children=[] if hl.annotation is None else
[node(heading=hl.annotation, )])
for hl in page.highlights
])
def get_items(self) -> Mirror.Results:
# TODO just use events? but need to sort first..
for e in github.get_events():
if isinstance(e, Exception):
yield error(e)
continue
# TODO filter only events that have body? e.g. not sure if much point emitting pull requests here
summary = e.summary
body = e.body
if body is None:
lines = summary.splitlines(keepends=True)
if len(lines) > 1:
summary = lines[0].strip()
body = ''.join(
lines[1:]
) # todo meh. hacky, better to extract bodies in the provider properly
if body.strip() == '':
body = None
yield node(
dt_heading(
e.dt,
link(url=e.link, title=summary)
if e.link is not None else summary),
body=None if body is None else pandoc.to_org(
body, from_='gfm'), # github flavored markdown
)
def get_items(self):
for page in get_pages():
# TODO would be nice to signal error upwards? Maybe just yield Exception, rener it in Orger and allow setting error status?
if isinstance(page, Exception):
yield error(page)
continue
yield node(
heading=dt_heading(page.created, link(title=page.title, url=page.url)),
children=[node(
heading=dt_heading(hl.created, link(title='context', url=hl.hyp_link)),
tags=hl.tags,
body=hl.highlight,
children=[] if hl.annotation is None else [node(
heading=hl.annotation,
)]
) for hl in page.highlights]
)
def get_items(self):
"""
get_items returns a sequence/iterator of nodes
see orger.inorganic.OrgNode to find out about attributes you can use
"""
export_file = self.cmdline_args.file # see setup_parser
for a in get_articles(export_file):
yield node(
heading=dt_heading(a.added, link(title=a.title, url=a.url)),
body=link(
title='Pocket link', url=a.pocket_link
), # permalink is pretty convenient to jump straight into Pocket app
children=[
node( # comments are displayed as org-mode child entries
heading=dt_heading(hl.created, hl.text))
for hl in a.highlights
])
def get_items(self):
for s in saves():
yield s.uid, node(
heading=dt_heading(
s.when,
link(title=s.title, url=s.hackernews_link),
),
body=s.url,
)
def get_items(self):
for s in get_saves():
yield s.sid, node(
# need to make heading lazy due to is_alive
heading=lambda s=s: dt_heading(s.created, (
'[#A] *DEAD*' if self.is_dead_url(s.url) else '') + link(
title=s.title, url=s.url) + f' /r/{s.subreddit}'),
body=s.text,
)
def get_items(self):
for s in saved():
yield s.sid, node(
# need to make heading lazy due to is_alive
# eh, can't guess the type of lambda??
heading=lambda s=s: dt_heading( # type: ignore[misc]
s.created,
('[#A] *DEAD*' if self.is_dead_url(s.url) else '') + link(title=s.title, url=s.url) + f' /r/{s.subreddit}'
),
body=s.text,
)
def get_items(self) -> Mirror.Results:
# TODO adapt for other stackexchange items
se_data = se.site('stackoverflow')
for q in se_data.questions:
# TODO could emit items along with level, then it would look a bit more natural
yield node(
dt_heading(q.creation_date, link(url=q.link, title=q.title)),
tags=q.tags,
body=Quoted(q.body_markdown),
# todo eh, would be useful to have md2org perhaps?
)
def get_items(self):
# TODO adapt for other stackexchange items
se_data = se.get_data()
for q in se_data.questions:
# TODO could emit items along with level, then it would look a bit more natural
yield node(
dt_heading(q.creation_date, link(url=q.link, title=q.title)),
tags=q.tags,
body=q.
body_markdown, # TODO eh, would be useful to have md2org perhaps?
)
def get_items(self):
for e in gh.get_events():
# TODO filter only events that have body? e.g. not sure if much point emitting pull requests here
yield node(
dt_heading(
e.dt,
link(url=e.link, title=e.summary)
if e.link is not None else e.summary),
# TODO would be nice to convert from markdown to org here
body=e.body,
)
def get_items(self) -> Queue.Results:
for page in pages():
bm = page.bookmark
for hl in page.highlights:
if not is_todo(hl):
continue
yield hl.hid, todo(
# todo dt_heading? not sure. Maybe this should be configurable
dt=hl.dt,
heading=link(title="X", url=hl.instapaper_link) + ' ' + hl.text,
tags=['ip2org'],
body=f'{hl.note}\nfrom {link(title=bm.title, url=bm.url)}',
)
def get_items(self) -> Mirror.Results:
for f in favorites():
if isinstance(f, Favorite):
yield node(
heading=dt_heading(
f.dt,
f.title if f.url is None else link(url=f.url,
title=f.title),
),
body=Quoted(f.text),
)
else: # Exception
yield error(f)
def get_items(self) -> Mirror.Results:
by_url = lambda w: w.url
by_when = lambda w: w.when
items = [
max(group, key=by_when)
for _, group in groupby(sorted(watched(), key=by_url), key=by_url)
]
items = sorted(items, key=by_when)
# TODO for each url only take latest?
for item in items:
deleted = item.url == item.title # todo move to HPI?
l = link(title=item.title + (' (DELETED)' if deleted else ''),
url=item.url)
yield (item.url, node(heading=dt_heading(item.when, l), ))
def get_items(self):
watched = get_watched()
by_url = lambda w: w.url
by_when = lambda w: w.when
items = [
max(group, key=by_when)
for _, group in groupby(sorted(watched, key=by_url), key=by_url)
]
items = sorted(items, key=by_when)
# TODO for each url only take latest?
for item in items:
yield (item.url, node(
heading=dt_heading(item.when, link(title=item.title, url=item.url)),
))
def roam_note_to_org(node: roamresearch.Node, top=False) -> Iterable[OrgNode]:
"""
Converts Roam node into Org-mode representation
"""
children = list(chain.from_iterable(map(roam_note_to_org, node.children)))
empty = len(node.body or '') == 0 and len(children) == 0
if empty:
# sometimes nodes are empty. two cases:
# - no heading -- child notes, like accidental enter presses I guess
# - heading -- notes that haven't been created yet
# just don't do anything in this case
# todo make this logic conditional?
return
title = node.title
# org-mode target allows jumping straight into
# conveniently, links in Roam are already represented as [[link]] !
target = '' if title is None else f'<<{title}>> '
heading = target + link(title='x', url=node.permalink)
todo = None
body = node.body
if body is not None:
for t in ('TODO', 'DONE'):
ts = '{{[[' + t + ']]}}'
if body.startswith(ts):
todo = t
body = body[len(ts):]
body = roam_text_to_org(body)
lines = body.splitlines(keepends=True)
# display first link of the body as the heading
if len(lines) > 0:
heading = heading + ' ' + lines[0]
body = ''.join(lines[1:])
if len(body) == 0:
body = None
if top:
heading = dt_heading(node.created, heading)
yield OrgNode(
todo=todo,
heading=heading,
body=body,
children=children,
)
def get_items(self):
for page in get_pages():
yield node(
heading=dt_heading(
page.dt,
f'{link(title="ip", url=page.bookmark.instapaper_link)} {link(title=page.title, url=page.url)}',
),
children=[node(
heading=dt_heading(hl.dt, link(title="ip", url=page.bookmark.instapaper_link)),
body=hl.text,
children=[] if hl.note is None else [
node(heading=hl.note),
],
) for hl in page.highlights]
)
def get_items(self):
for t in highlights():
if isinstance(t, Exception):
# I guess there isn't much we can do here? will be spotted by other tools
continue
if not is_todo(t):
continue
ann = t.annotation
anns = '' if ann is None else ann + '\n'
yield t.hid, todo(
dt=t.created,
heading=link(title="X", url=t.hyp_link) + ' ' + t.highlight,
body=f'{anns}from {link(title=t.title, url=t.url)}',
tags=['hyp2org', *t.tags],
)
def _render(self, t: Tweet) -> OrgNode:
dtime = t.created_at
text = t.text
url = t.permalink
if self.mode == 'all':
return node(
heading=dt_heading(dtime, link(title=text, url=url)),
tags=['tweet'],
)
else:
dd = dtime.replace(year=today.year)
text = ' ' + text if text.startswith('@') else text # pad replies a bit
return node(
heading=timestamp(dd.date(), active=True) + ' ' + f"{link(title='TW', url=url)} at {timestamp(dtime, inactive=True)} {text}",
tags=['ttweet'],
)
def get_items(self):
for f in favorites():
if isinstance(f, Favorite):
yield node(
heading=dt_heading(
f.dt,
f.title if f.url is None else link(url=f.url,
title=f.title),
),
body=f.text,
)
else: # error
yield node(
heading=f'ERROR!',
body='\n'.join(
traceback.format_exception(Exception, f, None)),
)
def get_items(self) -> Queue.Results:
for t in highlights():
if isinstance(t, Exception):
# todo make error helper work with Queue
# probably hash of the body would be enough? dunno
# I guess there isn't much we can do here? will be spotted by other tools
continue
if not is_todo(t):
continue
ann = t.annotation
anns = '' if ann is None else ann + '\n'
hl = t.highlight or ''
yield t.hid, todo(
dt=t.created,
heading=link(title="X", url=t.hyp_link) + ' ' + hl,
body=f'{anns}from {link(title=t.title, url=t.url)}',
# todo maybe get rid of hyp2org?
# or make tagging an option?
tags=['hyp2org', *t.tags],
)
def make_item(res: polar.Result):
if isinstance(res, polar.Error):
# TODO could create error heading from exception automatically? take first line as heading and rest + traceback as the body
return node(heading='ERROR ' + str(res)) # TODO priority A?
else:
book = res
return node(
heading=dt_heading(
book.created,
# TODO apparently file: is not necessary if the path is absolute?
link(url=str(book.path), title=book.title),
),
tags=book.tags,
children=[
node(
heading=dt_heading(hl.created, hl.selection),
tags=hl.tags,
properties=None if hl.color is None else
{'POLAR_COLOR': hex2name(hl.color)},
children=[make_comment(c) for c in hl.comments],
) for hl in book.items
])
def get_items(self):
for s in rss.subscriptions():
yield node(
link(url=s.url, title=s.title) + ('' if s.subscribed else ' UNSUBSCRIBED'),
)
def format_group(group: List, dialog,
logger) -> Tuple[Timestamp, From, Tags, Lines]:
date = int(group[0].date.timestamp())
def get_from(m):
fw = m.forward
if fw is None:
return 'me'
if fw.sender is None:
if fw.chat is not None:
return fw.chat.title
else:
return "ERROR UNKNOWN SENDER"
u = fw.sender
if u.username is not None:
return u.username
else:
return f"{u.first_name} {u.last_name}"
froms = [get_from(m) for m in group]
tags = {NAME_TO_TAG[f] for f in froms if f in NAME_TO_TAG}
from_ = ', '.join(
link(url=f'https://t.me/{f}', title=f) for f in sorted(set(froms)))
texts: List[str] = []
for m in group:
texts.append(m.message)
# TODO hmm, _text contains markdown? could convert it to org...
# TODO use m.entities??
if m.media is None:
continue
e = m.media
if isinstance(e, MessageMediaWebPage):
page = e.webpage
uu: str
if isinstance(page, WebPageEmpty):
uu = "*empty web page*"
else:
title = page.display_url if page.title is None else page.title
uu = link(url=page.url, title=title)
if page.description is not None:
uu += ' ' + page.description
texts.append(uu)
elif isinstance(e, MessageMediaPhoto):
# TODO no file location? :(
texts.append("*PHOTO*")
# print(vars(e))
elif isinstance(e, MessageMediaDocument):
texts.append("*DOCUMENT*")
# print(vars(e.document))
elif isinstance(e, MessageMediaVenue):
texts.append("*VENUE* " + e.title)
else:
logger.error(f"Unknown media {type(e)}")
# raise RuntimeError
# TODO contribute 1 to exit code? or emit Error?
# chat = dialog.name
# mid = group[0].id
# TODO ugh. doesn't seem to be possible to jump to private dialog :(
# and couldn't get original forwarded message id from message object..
# in_context = f'https://t.me/{chat}/{mid}'
# TODO detect by data-msg-id?
texts = list(reversed(texts))
heading = from_
LIMIT = 400
lines = '\n'.join(texts).splitlines() # meh
for line in lines:
if len(heading) + len(line) <= LIMIT:
heading += " " + line
else:
break
heading = re.sub(r'\s', ' ', heading) # TODO rely on inorganic for that?
return (date, heading, tags, texts)
def get_items(self):
for tweet in twi.likes():
# likes don't have timestamps (at least from GDPR export data)
# TODO support it and handle None
yield node(link(url=tweet.permalink, title='liked'),
body=tweet.text)
def make_item(b: pinboard.Bookmark):
return node(
heading=dt_heading(b.created, link(title=b.title, url=b.url)),
body=b.description,
tags=b.tags,
)
