def render():
"""Fetches a stored Response and renders it as HTML."""
source = flask_util.get_required_param('source')
target = flask_util.get_required_param('target')
id = f'{source} {target}'
resp = Response.get_by_id(id)
if not resp:
error(f'No stored response for {id}', status=404)
if resp.source_mf2:
as1 = microformats2.json_to_object(json_loads(resp.source_mf2))
elif resp.source_as2:
as1 = as2.to_as1(json_loads(resp.source_as2))
elif resp.source_atom:
as1 = atom.atom_to_activity(resp.source_atom)
else:
error(f'Stored response for {id} has no data', status=404)
# add HTML meta redirect to source page. should trigger for end users in
# browsers but not for webmention receivers (hopefully).
html = microformats2.activities_to_html([as1])
utf8 = '<meta charset="utf-8">'
refresh = f'<meta http-equiv="refresh" content="0;url={source}">'
return html.replace(utf8, utf8 + '\n' + refresh)
def test_activities_to_html_like(self):
self.assert_multiline_equals(
"""\
<!DOCTYPE html>
<html>
<head><meta charset="utf-8"></head>
<body>
<article class="h-entry">
<span class="p-uid">http://localhost:3000/users/ryan#likes/7</span>
<span class="p-author h-card">
<a class="u-url" href="http://localhost:3000/users/ryan">http://localhost:3000/users/ryan</a>
</span>
<div class="e-content p-name">
<a href="http://localhost/2017-10-01_mastodon-dev-6">likes this.</a>
</div>
<a class="u-like-of" href="http://localhost/2017-10-01_mastodon-dev-6"></a>
</article>
</body>
</html>
""",
microformats2.activities_to_html([{
'id': 'http://localhost:3000/users/ryan#likes/7',
'objectType': 'activity',
'verb': 'like',
'object': {
'url': 'http://localhost/2017-10-01_mastodon-dev-6'
},
'actor': {
'url': 'http://localhost:3000/users/ryan'
},
}]),
ignore_blanks=True)
def render(activities, actor=None):
# Pass images and videos through caching proxy to cache them
for a in activities:
microformats2.prefix_image_urls(a, IMAGE_PROXY_URL_BASE)
microformats2.prefix_video_urls(a, VIDEO_PROXY_URL_BASE)
# Generate output
format = request.args.get('format') or 'atom'
if format == 'atom':
title = 'instagram-atom feed for %s' % source.Source.actor_name(actor)
return atom.activities_to_atom(
activities,
actor,
title=title,
host_url=request.host_url,
request_url=request.url,
xml_base='https://www.instagram.com/',
), {
'Content-Type': 'application/atom+xml'
}
elif format == 'html':
return microformats2.activities_to_html(activities)
else:
flask_util.error(f'format must be either atom or html; got {format}')
def test_activities_to_html_like(self):
self.assert_multiline_equals("""\
<!DOCTYPE html>
<html>
<head><meta charset="utf-8"></head>
<body>
<article class="h-entry">
<span class="p-uid">http://localhost:3000/users/ryan#likes/7</span>
<span class="p-author h-card">
<a class="u-url" href="http://localhost:3000/users/ryan">http://localhost:3000/users/ryan</a>
</span>
<div class="e-content p-name">
<a href="http://localhost/2017-10-01_mastodon-dev-6">likes this.</a>
</div>
<a class="u-like-of" href="http://localhost/2017-10-01_mastodon-dev-6"></a>
</article>
</body>
</html>
""", microformats2.activities_to_html([{
'id': 'http://localhost:3000/users/ryan#likes/7',
'objectType': 'activity',
'verb': 'like',
'object': {'url': 'http://localhost/2017-10-01_mastodon-dev-6'},
'actor': {'url': 'http://localhost:3000/users/ryan'},
}]), ignore_blanks=True)
def test_share_activity_to_json_html(self):
"""Should translate the full activity, not just the object."""
share = {
'verb': 'share',
'actor': {'displayName': 'sharer'},
'object': {
'content': 'original',
'actor': {'displayName': 'author'},
},
}
self.assert_equals({
'type': ['h-entry'],
'properties': {
'author': [{
'type': ['h-card'],
'properties': {'name': ['sharer']},
}],
'repost-of': [{
'type': ['h-cite'],
'properties': {
'content': [{'html': 'original', 'value': 'original'}],
'author': [{
'type': ['h-card'],
'properties': {'name': ['author']},
}],
}
}],
},
}, microformats2.activity_to_json(share, synthesize_content=False))
self.assert_multiline_in("""\
Shared <a href="#">a post</a> by <span class="h-card">
<span class="p-name">author</span>
""", microformats2.activities_to_html([share]), ignore_blanks=True)
def get(self):
source = util.get_required_param(self, 'source')
target = util.get_required_param(self, 'target')
id = '%s %s' % (source, target)
resp = Response.get_by_id(id)
if not resp:
self.abort(404, 'No stored response for %s' % id)
if resp.source_mf2:
as1 = microformats2.json_to_object(json.loads(resp.source_mf2))
elif resp.source_as2:
as1 = as2.to_as1(json.loads(resp.source_as2))
elif resp.source_atom:
as1 = atom.atom_to_activity(resp.source_atom)
else:
self.abort(404, 'Stored response for %s has no data' % id)
# add HTML meta redirect to source page. should trigger for end users in
# browsers but not for webmention receivers (hopefully).
html = microformats2.activities_to_html([as1])
utf8 = '<meta charset="utf-8">'
refresh = '<meta http-equiv="refresh" content="0;url=%s">' % source
html = html.replace(utf8, utf8 + '\n' + refresh)
self.response.write(html)
def write_response(self, response, actor=None, url=None, title=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom output.
url: the input URL
title: string, Used in Atom output
"""
expected_formats = ('activitystreams', 'json', 'atom', 'xml', 'html', 'json-mf2')
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in expected_formats:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, expected_formats))
activities = response['items']
self.response.headers.update({
'Access-Control-Allow-Origin': '*',
'Strict-Transport-Security':
'max-age=16070400; includeSubDomains; preload', # 6 months
})
if format in ('json', 'activitystreams'):
# list of official MIME types:
# https://www.iana.org/assignments/media-types/media-types.xhtml
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'application/atom+xml'
hub = self.request.get('hub')
self.response.out.write(atom.activities_to_atom(
activities, actor,
host_url=url or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None))
self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'xml':
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(microformats2.activities_to_html(activities))
elif format == 'json-mf2':
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.object_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def write_response(self, response, actor=None, url=None, title=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom output.
url: the input URL
title: string, Used in Atom output
"""
expected_formats = ('activitystreams', 'json', 'atom', 'xml', 'html', 'json-mf2')
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in expected_formats:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, expected_formats))
activities = response['items']
self.response.headers.update({
'Access-Control-Allow-Origin': '*',
'Strict-Transport-Security':
'max-age=16070400; includeSubDomains; preload', # 6 months
})
if format in ('json', 'activitystreams'):
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'text/xml'
hub = self.request.get('hub')
self.response.out.write(atom.activities_to_atom(
activities, actor,
host_url=url or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None))
self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'xml':
self.response.headers['Content-Type'] = 'text/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(microformats2.activities_to_html(activities))
elif format == 'json-mf2':
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.object_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def test_share_activity_to_json_html(self):
"""Should translate the full activity, not just the object."""
share = {
'verb': 'share',
'actor': {
'displayName': 'sharer'
},
'object': {
'content': 'original',
'actor': {
'displayName': 'author'
},
},
}
self.assert_equals(
{
'type': ['h-entry'],
'properties': {
'author': [{
'type': ['h-card'],
'properties': {
'name': ['sharer']
},
}],
'repost-of': [{
'type': ['h-cite'],
'properties': {
'content': [{
'html': 'original',
'value': 'original'
}],
'author': [{
'type': ['h-card'],
'properties': {
'name': ['author']
},
}],
}
}],
},
}, microformats2.activity_to_json(share, synthesize_content=False))
self.assert_multiline_in("""\
Shared <a href="#">a post</a> by <span class="h-card">
<span class="p-name">author</span>
""",
microformats2.activities_to_html([share]),
ignore_blanks=True)
def get(self):
source = util.get_required_param(self, 'source')
target = util.get_required_param(self, 'target')
id = '%s %s' % (source, target)
resp = Response.get_by_id(id)
if not resp:
self.abort(404, 'No stored response for %s' % id)
if resp.source_mf2:
as1 = microformats2.json_to_object(json.loads(resp.source_mf2))
elif resp.source_as2:
as1 = as2.to_as1(json.loads(resp.source_as2))
elif resp.source_atom:
as1 = atom.atom_to_activity(resp.source_atom)
else:
self.abort(404, 'Stored response for %s has no data' % id)
self.response.write(microformats2.activities_to_html([as1]))
def write_response(self, response, actor=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom output.
"""
expected_formats = ('activitystreams', 'json', 'atom', 'xml', 'html', 'json-mf2')
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in expected_formats:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, expected_formats))
activities = response['items']
self.response.headers['Access-Control-Allow-Origin'] = '*'
if format in ('json', 'activitystreams'):
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'text/xml'
self.response.out.write(atom.activities_to_atom(
activities, actor, host_url=self.request.host_url + '/',
request_url=self.request.path_url))
elif format == 'xml':
self.response.headers['Content-Type'] = 'text/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(microformats2.activities_to_html(activities))
elif format == 'json-mf2':
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.object_to_json(a['object'], a.get('context', {}))
for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def write_response(self,
response,
actor=None,
url=None,
title=None,
hfeed=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, used in feed output (Atom, JSON Feed, RSS)
hfeed: dict, parsed mf2 h-feed, if available
"""
format = self.request.get('format') or self.request.get(
'output') or 'json'
if format not in FORMATS:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, FORMATS))
if 'plaintext' in self.request.params:
# override content type
self.response.headers['Content-Type'] = 'text/plain'
else:
content_type = FORMATS.get(format)
if content_type:
self.response.headers['Content-Type'] = content_type
if self.request.method == 'HEAD':
return
activities = response['items']
try:
if format in ('as1', 'json', 'activitystreams'):
self.response.out.write(json_dumps(response, indent=2))
elif format == 'as2':
response.update({
'items': [as2.from_as1(a) for a in activities],
'totalItems':
response.pop('totalResults', None),
'updated':
response.pop('updatedSince', None),
'filtered':
None,
'sorted':
None,
})
self.response.out.write(
json_dumps(util.trim_nulls(response), indent=2))
elif format == 'atom':
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400,
'reader param must be either true or false')
if not actor and hfeed:
actor = microformats2.json_to_object({
'properties':
hfeed.get('properties', {}),
})
self.response.out.write(
atom.activities_to_atom(activities,
actor,
host_url=url
or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add(
'Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link',
str('<%s>; rel="hub"' % hub))
elif format == 'rss':
if not title:
title = 'Feed for %s' % url
self.response.out.write(
rss.from_activities(activities,
actor,
title=title,
feed_url=self.request.url,
hfeed=hfeed,
home_page_url=util.base_url(url)))
elif format in ('as1-xml', 'xml'):
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.out.write(
microformats2.activities_to_html(activities))
elif format in ('mf2-json', 'json-mf2'):
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json_dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
try:
jf = jsonfeed.activities_to_jsonfeed(
activities,
actor=actor,
title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json_dumps(jf, indent=2))
except ValueError as e:
logging.warning('converting to output format failed',
stack_info=True)
self.abort(400, 'Could not convert to %s: %s' % (format, str(e)))
def write_response(self, response, actor=None, url=None, title=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, Used in Atom and JSON Feed output
"""
expected_formats = ('activitystreams', 'json', 'atom', 'xml', 'html',
'json-mf2', 'jsonfeed')
format = self.request.get('format') or self.request.get(
'output') or 'json'
if format not in expected_formats:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, expected_formats))
activities = response['items']
if format in ('json', 'activitystreams'):
# list of official MIME types:
# https://www.iana.org/assignments/media-types/media-types.xhtml
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'application/atom+xml'
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400, 'reader param must be either true or false')
self.response.out.write(
atom.activities_to_atom(activities,
actor,
host_url=url
or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add(
'Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'xml':
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(
microformats2.activities_to_html(activities))
elif format == 'json-mf2':
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
self.response.headers['Content-Type'] = 'application/json'
try:
jf = jsonfeed.activities_to_jsonfeed(activities,
actor=actor,
title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json.dumps(jf, indent=2))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def write_response(self, response, actor=None, url=None, title=None,
hfeed=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, used in feed output (Atom, JSON Feed, RSS)
hfeed: dict, parsed mf2 h-feed, if available
"""
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in FORMATS:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, FORMATS))
activities = response['items']
try:
if format in ('as1', 'json', 'activitystreams'):
# list of official MIME types:
# https://www.iana.org/assignments/media-types/media-types.xhtml
self.response.headers['Content-Type'] = \
'application/json' if format == 'json' else 'application/stream+json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'as2':
self.response.headers['Content-Type'] = 'application/activity+json'
response.update({
'items': [as2.from_as1(a) for a in activities],
'totalItems': response.pop('totalResults', None),
'updated': response.pop('updatedSince', None),
'filtered': None,
'sorted': None,
})
self.response.out.write(json.dumps(util.trim_nulls(response), indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'application/atom+xml'
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400, 'reader param must be either true or false')
self.response.out.write(atom.activities_to_atom(
activities, actor,
host_url=url or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'rss':
self.response.headers['Content-Type'] = 'application/rss+xml'
if not title:
title = 'Feed for %s' % url
self.response.out.write(rss.from_activities(
activities, actor, title=title,
feed_url=self.request.url, hfeed=hfeed,
home_page_url=util.base_url(url)))
elif format in ('as1-xml', 'xml'):
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(microformats2.activities_to_html(activities))
elif format in ('mf2-json', 'json-mf2'):
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
self.response.headers['Content-Type'] = 'application/json'
try:
jf = jsonfeed.activities_to_jsonfeed(activities, actor=actor, title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json.dumps(jf, indent=2))
except ValueError as e:
logging.warning('converting to output format failed', exc_info=True)
self.abort(400, 'Could not convert to %s: %s' % (format, str(e)))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def get(self):
cookie = 'sessionid=%s' % urllib.parse.quote(
util.get_required_param(self, 'sessionid').encode('utf-8'))
logging.info('Fetching with Cookie: %s', cookie)
host_url = self.request.host_url + '/'
ig = instagram.Instagram()
try:
resp = ig.get_activities_response(group_id=source.FRIENDS,
scrape=True,
cookie=cookie)
except Exception as e:
status, text = util.interpret_http_exception(e)
if status in ('403', ):
self.response.headers['Content-Type'] = 'application/atom+xml'
self.response.out.write(
atom.activities_to_atom([{
'object': {
'url':
self.request.url,
'content':
'Your instagram-atom cookie isn\'t working. <a href="%s">Click here to regenerate your feed!</a>'
% host_url,
},
}], {},
title='instagram-atom',
host_url=host_url,
request_url=self.request.path_url))
return
elif status == '401':
# IG returns 401 sometimes as a form of rate limiting or bot detection
self.response.status = '429'
elif status:
self.response.status = status
else:
logging.exception('oops!')
self.response.status = 500
self.response.text = text or 'Unknown error.'
return
actor = resp.get('actor')
if actor:
logging.info('Logged in as %s (%s)', actor.get('username'),
actor.get('displayName'))
else:
logging.warning("Couldn't determine Instagram user!")
activities = resp.get('items', [])
format = self.request.get('format', 'atom')
if format == 'atom':
title = 'instagram-atom feed for %s' % ig.actor_name(actor)
self.response.headers['Content-Type'] = 'application/atom+xml'
self.response.out.write(
atom.activities_to_atom(activities,
actor,
title=title,
host_url=host_url,
request_url=self.request.path_url,
xml_base='https://www.instagram.com/'))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(
microformats2.activities_to_html(activities))
else:
self.abort(400,
'format must be either atom or html; got %s' % format)