def test_bad_input_types(self):
for bad in 1, [2], (3, ):
for fn in as2.to_as1, as2.from_as1:
with self.assertRaises(ValueError):
fn(bad)
with self.assertRaises(ValueError):
as2.from_as1('z')
def test_bad_input_types(self):
for bad in 1, [2], (3,):
for fn in as2.to_as1, as2.from_as1:
with self.assertRaises(ValueError):
fn(bad)
with self.assertRaises(ValueError):
as2.from_as1('z')
def get(self, domain):
url = 'http://%s/' % domain
resp = common.requests_get(url)
mf2 = mf2py.parse(resp.text, url=resp.url, img_with_alt=True)
# logging.info('Parsed mf2 for %s: %s', resp.url, json.dumps(mf2, indent=2))
hcard = mf2util.representative_hcard(mf2, resp.url)
logging.info('Representative h-card: %s', json.dumps(hcard, indent=2))
if not hcard:
common.error(
self, """\
Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s"""
% resp.url)
key = MagicKey.get_or_create(domain)
obj = common.postprocess_as2(as2.from_as1(
microformats2.json_to_object(hcard)),
key=key)
obj.update({
'inbox':
'%s/%s/inbox' % (appengine_config.HOST_URL, domain),
'outbox':
'%s/%s/outbox' % (appengine_config.HOST_URL, domain),
'following':
'%s/%s/following' % (appengine_config.HOST_URL, domain),
'followers':
'%s/%s/followers' % (appengine_config.HOST_URL, domain),
})
logging.info('Returning: %s', json.dumps(obj, indent=2))
self.response.headers.update({
'Content-Type': common.CONTENT_TYPE_AS2,
'Access-Control-Allow-Origin': '*',
})
self.response.write(json.dumps(obj, indent=2))
def actor(domain):
"""Serves /[DOMAIN], fetches its mf2, converts to AS Actor, and serves it."""
tld = domain.split('.')[-1]
if tld in common.TLD_BLOCKLIST:
error('', status=404)
mf2 = util.fetch_mf2(f'http://{domain}/',
gateway=True,
headers=common.HEADERS)
hcard = mf2util.representative_hcard(mf2, mf2['url'])
logging.info(f'Representative h-card: {json_dumps(hcard, indent=2)}')
if not hcard:
error(
f"Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on {mf2['url']}"
)
key = MagicKey.get_or_create(domain)
obj = common.postprocess_as2(as2.from_as1(
microformats2.json_to_object(hcard)),
key=key)
obj.update({
'preferredUsername': domain,
'inbox': f'{request.host_url}{domain}/inbox',
'outbox': f'{request.host_url}{domain}/outbox',
'following': f'{request.host_url}{domain}/following',
'followers': f'{request.host_url}{domain}/followers',
})
logging.info(f'Returning: {json_dumps(obj, indent=2)}')
return (obj, {
'Content-Type': common.CONTENT_TYPE_AS2,
'Access-Control-Allow-Origin': '*',
})
def get(self, domain):
tld = domain.split('.')[-1]
if tld in common.TLD_BLOCKLIST:
self.error('', status=404)
mf2 = util.fetch_mf2('http://%s/' % domain, gateway=True,
headers=common.HEADERS)
# logging.info('Parsed mf2 for %s: %s', resp.url, json_dumps(mf2, indent=2))
hcard = mf2util.representative_hcard(mf2, mf2['url'])
logging.info('Representative h-card: %s', json_dumps(hcard, indent=2))
if not hcard:
self.error("""\
Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % mf2['url'])
key = MagicKey.get_or_create(domain)
obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)),
key=key)
obj.update({
'inbox': '%s/%s/inbox' % (self.request.host_url, domain),
'outbox': '%s/%s/outbox' % (self.request.host_url, domain),
'following': '%s/%s/following' % (self.request.host_url, domain),
'followers': '%s/%s/followers' % (self.request.host_url, domain),
})
logging.info('Returning: %s', json_dumps(obj, indent=2))
self.response.headers.update({
'Content-Type': common.CONTENT_TYPE_AS2,
'Access-Control-Allow-Origin': '*',
})
self.response.write(json_dumps(obj, indent=2))
def try_activitypub(self):
"""Returns True if we attempted ActivityPub delivery, False otherwise."""
targets = self._activitypub_targets()
if not targets:
return False
key = MagicKey.get_or_create(self.source_domain)
error = None
last_success = None
# TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients
for resp, inbox in targets:
target_obj = json_loads(
resp.target_as2) if resp.target_as2 else None
source_activity = self.postprocess_as2(as2.from_as1(
self.source_obj),
target=target_obj,
key=key)
if resp.status == 'complete':
source_activity['type'] = 'Update'
try:
last = activitypub.send(source_activity, inbox,
self.source_domain)
resp.status = 'complete'
last_success = last
except BaseException as e:
error = e
resp.status = 'error'
resp.put()
# Pass the AP response status code and body through as our response
if last_success:
self.response.status_int = last_success.status_code
self.response.write(last_success.text)
elif isinstance(error, requests.HTTPError):
self.response.status_int = error.status_code
self.response.write(error.text)
else:
self.response.write(str(error))
return bool(last_success)
def convert_to_as2(url):
"""Fetch a URL as HTML, convert it to AS2, and return it.
Currently mainly for Pixelfed.
https://github.com/snarfed/bridgy-fed/issues/39
"""
mf2 = util.fetch_mf2(url)
entry = mf2util.find_first_entry(mf2, ['h-entry'])
logging.info(f"Parsed mf2 for {mf2['url']}: {json_dumps(entry, indent=2)}")
obj = common.postprocess_as2(
as2.from_as1(microformats2.json_to_object(entry)))
logging.info(f'Returning: {json_dumps(obj, indent=2)}')
return obj, {
'Content-Type': common.CONTENT_TYPE_AS2,
'Access-Control-Allow-Origin': '*',
}
def convert_to_as2(self, url):
"""Fetch a URL as HTML, convert it to AS2, and return it.
Currently mainly for Pixelfed.
https://github.com/snarfed/bridgy-fed/issues/39
"""
mf2 = util.fetch_mf2(url)
entry = mf2util.find_first_entry(mf2, ['h-entry'])
logging.info('Parsed mf2 for %s: %s', mf2['url'],
json_dumps(entry, indent=2))
obj = self.postprocess_as2(
as2.from_as1(microformats2.json_to_object(entry)))
logging.info('Returning: %s', json_dumps(obj, indent=2))
self.response.headers.update({
'Content-Type': common.CONTENT_TYPE_AS2,
'Access-Control-Allow-Origin': '*',
})
self.response.write(json_dumps(obj, indent=2))
def test_from_as1_blank(self):
self.assertEqual({}, as2.from_as1(None))
self.assertEqual({}, as2.from_as1({}))
def write_response(self,
response,
actor=None,
url=None,
title=None,
hfeed=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, used in feed output (Atom, JSON Feed, RSS)
hfeed: dict, parsed mf2 h-feed, if available
"""
format = self.request.get('format') or self.request.get(
'output') or 'json'
if format not in FORMATS:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, FORMATS))
if 'plaintext' in self.request.params:
# override content type
self.response.headers['Content-Type'] = 'text/plain'
else:
content_type = FORMATS.get(format)
if content_type:
self.response.headers['Content-Type'] = content_type
if self.request.method == 'HEAD':
return
activities = response['items']
try:
if format in ('as1', 'json', 'activitystreams'):
self.response.out.write(json_dumps(response, indent=2))
elif format == 'as2':
response.update({
'items': [as2.from_as1(a) for a in activities],
'totalItems':
response.pop('totalResults', None),
'updated':
response.pop('updatedSince', None),
'filtered':
None,
'sorted':
None,
})
self.response.out.write(
json_dumps(util.trim_nulls(response), indent=2))
elif format == 'atom':
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400,
'reader param must be either true or false')
if not actor and hfeed:
actor = microformats2.json_to_object({
'properties':
hfeed.get('properties', {}),
})
self.response.out.write(
atom.activities_to_atom(activities,
actor,
host_url=url
or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add(
'Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link',
str('<%s>; rel="hub"' % hub))
elif format == 'rss':
if not title:
title = 'Feed for %s' % url
self.response.out.write(
rss.from_activities(activities,
actor,
title=title,
feed_url=self.request.url,
hfeed=hfeed,
home_page_url=util.base_url(url)))
elif format in ('as1-xml', 'xml'):
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.out.write(
microformats2.activities_to_html(activities))
elif format in ('mf2-json', 'json-mf2'):
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json_dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
try:
jf = jsonfeed.activities_to_jsonfeed(
activities,
actor=actor,
title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json_dumps(jf, indent=2))
except ValueError as e:
logging.warning('converting to output format failed',
stack_info=True)
self.abort(400, 'Could not convert to %s: %s' % (format, str(e)))
def try_activitypub(self):
source = util.get_required_param(self, 'source')
# fetch source page, convert to ActivityStreams
source_resp = common.requests_get(source)
source_url = source_resp.url or source
source_mf2 = mf2py.parse(source_resp.text, url=source_url)
# logging.debug('Parsed mf2 for %s: %s', source_resp.url, json.dumps(source_mf2, indent=2))
entry = mf2util.find_first_entry(source_mf2, ['h-entry'])
logging.info('First entry: %s', json.dumps(entry, indent=2))
# make sure it has url, since we use that for AS2 id, which is required
# for ActivityPub.
props = entry.setdefault('properties', {})
if not props.get('url'):
props['url'] = [source_url]
source_obj = microformats2.json_to_object(entry, fetch_mf2=True)
logging.info('Converted to AS: %s', json.dumps(source_obj, indent=2))
# fetch target page as AS object. target is first in-reply-to, like-of,
# or repost-of, *not* target query param.)
target = util.get_url(util.get_first(source_obj, 'inReplyTo') or
util.get_first(source_obj, 'object'))
if not target:
common.error(self, 'No u-in-reply-to, u-like-of, or u-repost-of '
'found in %s' % source_url)
try:
target_resp = common.get_as2(target)
except (requests.HTTPError, exc.HTTPBadGateway) as e:
if (e.response.status_code // 100 == 2 and
common.content_type(e.response).startswith('text/html')):
self.resp = Response.get_or_create(
source=source_url, target=e.response.url or target,
direction='out', source_mf2=json.dumps(source_mf2))
return self.send_salmon(source_obj, target_resp=e.response)
raise
target_url = target_resp.url or target
self.resp = Response.get_or_create(
source=source_url, target=target_url, direction='out',
protocol='activitypub', source_mf2=json.dumps(source_mf2))
# find actor's inbox
target_obj = target_resp.json()
inbox_url = target_obj.get('inbox')
if not inbox_url:
# TODO: test actor/attributedTo and not, with/without inbox
actor = target_obj.get('actor') or target_obj.get('attributedTo')
if isinstance(actor, dict):
inbox_url = actor.get('inbox')
actor = actor.get('url')
if not inbox_url and not actor:
common.error(self, 'Target object has no actor or attributedTo URL')
if not inbox_url:
# fetch actor as AS object
actor = common.get_as2(actor).json()
inbox_url = actor.get('inbox')
if not inbox_url:
# TODO: probably need a way to save errors like this so that we can
# return them if ostatus fails too.
# common.error(self, 'Target actor has no inbox')
return self.send_salmon(source_obj, target_resp=target_resp)
# convert to AS2
source_domain = urlparse.urlparse(source_url).netloc
key = MagicKey.get_or_create(source_domain)
source_activity = common.postprocess_as2(
as2.from_as1(source_obj), target=target_obj, key=key)
if self.resp.status == 'complete':
source_activity['type'] = 'Update'
# prepare HTTP Signature (required by Mastodon)
# https://w3c.github.io/activitypub/#authorization-lds
# https://tools.ietf.org/html/draft-cavage-http-signatures-07
# https://github.com/tootsuite/mastodon/issues/4906#issuecomment-328844846
acct = 'acct:%s@%s' % (source_domain, source_domain)
auth = HTTPSignatureAuth(secret=key.private_pem(), key_id=acct,
algorithm='rsa-sha256')
# deliver source object to target actor's inbox.
headers = {
'Content-Type': common.CONTENT_TYPE_AS2,
# required for HTTP Signature
# https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3
'Date': datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'),
}
inbox_url = urlparse.urljoin(target_url, inbox_url)
resp = common.requests_post(inbox_url, json=source_activity, auth=auth,
headers=headers)
self.response.status_int = resp.status_code
if resp.status_code == 202:
self.response.write('202 response! If this is Mastodon 1.x, their '
'signature verification probably failed. :(\n')
self.response.write(resp.text)
def try_activitypub(self):
"""Attempts ActivityPub delivery.
Returns Flask response (string body or tuple) if we succeeded or failed,
None if ActivityPub was not available.
"""
targets = self._activitypub_targets()
if not targets:
return None
key = MagicKey.get_or_create(self.source_domain)
error = None
last_success = None
# TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients
for resp, inbox in targets:
target_obj = json_loads(
resp.target_as2) if resp.target_as2 else None
source_activity = common.postprocess_as2(as2.from_as1(
self.source_obj),
target=target_obj,
key=key)
if resp.status == 'complete':
if resp.source_mf2:
def content(mf2):
items = mf2.get('items')
if items:
return microformats2.first_props(
items[0].get('properties')).get('content')
orig_content = content(json_loads(resp.source_mf2))
new_content = content(self.source_mf2)
if orig_content and new_content and orig_content == new_content:
msg = f'Skipping; new content is same as content published before at {resp.updated}'
logging.info(msg)
return msg
source_activity['type'] = 'Update'
try:
last = activitypub.send(source_activity, inbox,
self.source_domain)
resp.status = 'complete'
last_success = last
except BaseException as e:
error = e
resp.status = 'error'
resp.put()
# Pass the AP response status code and body through as our response
if last_success:
return last_success.text or 'Sent!', last_success.status_code
elif isinstance(error, BadGateway):
raise error
elif isinstance(error, requests.HTTPError):
return str(error), error.status_code
else:
return str(error)
def write_response(self, response, actor=None, url=None, title=None,
hfeed=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, used in feed output (Atom, JSON Feed, RSS)
hfeed: dict, parsed mf2 h-feed, if available
"""
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in FORMATS:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, FORMATS))
activities = response['items']
try:
if format in ('as1', 'json', 'activitystreams'):
# list of official MIME types:
# https://www.iana.org/assignments/media-types/media-types.xhtml
self.response.headers['Content-Type'] = \
'application/json' if format == 'json' else 'application/stream+json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'as2':
self.response.headers['Content-Type'] = 'application/activity+json'
response.update({
'items': [as2.from_as1(a) for a in activities],
'totalItems': response.pop('totalResults', None),
'updated': response.pop('updatedSince', None),
'filtered': None,
'sorted': None,
})
self.response.out.write(json.dumps(util.trim_nulls(response), indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'application/atom+xml'
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400, 'reader param must be either true or false')
self.response.out.write(atom.activities_to_atom(
activities, actor,
host_url=url or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'rss':
self.response.headers['Content-Type'] = 'application/rss+xml'
if not title:
title = 'Feed for %s' % url
self.response.out.write(rss.from_activities(
activities, actor, title=title,
feed_url=self.request.url, hfeed=hfeed,
home_page_url=util.base_url(url)))
elif format in ('as1-xml', 'xml'):
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(microformats2.activities_to_html(activities))
elif format in ('mf2-json', 'json-mf2'):
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
self.response.headers['Content-Type'] = 'application/json'
try:
jf = jsonfeed.activities_to_jsonfeed(activities, actor=actor, title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json.dumps(jf, indent=2))
except ValueError as e:
logging.warning('converting to output format failed', exc_info=True)
self.abort(400, 'Could not convert to %s: %s' % (format, str(e)))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def write_response(self, response, actor=None, url=None, title=None):
"""Converts ActivityStreams activities and writes them out.
Args:
response: response dict with values based on OpenSocial ActivityStreams
REST API, as returned by Source.get_activities_response()
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, Used in Atom and JSON Feed output
"""
format = self.request.get('format') or self.request.get(
'output') or 'json'
if format not in FORMATS:
raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' %
(format, FORMATS))
activities = response['items']
if format in ('as1', 'json', 'activitystreams'):
# list of official MIME types:
# https://www.iana.org/assignments/media-types/media-types.xhtml
self.response.headers['Content-Type'] = \
'application/json' if format == 'json' else 'application/stream+json'
self.response.out.write(json.dumps(response, indent=2))
elif format == 'as2':
self.response.headers['Content-Type'] = 'application/activity+json'
response.update({
'items': [as2.from_as1(a) for a in activities],
'totalItems': response.pop('totalResults', None),
'updated': response.pop('updatedSince', None),
'filtered': None,
'sorted': None,
})
self.response.out.write(
json.dumps(util.trim_nulls(response), indent=2))
elif format == 'atom':
self.response.headers['Content-Type'] = 'application/atom+xml'
hub = self.request.get('hub')
reader = self.request.get('reader', 'true').lower()
if reader not in ('true', 'false'):
self.abort(400, 'reader param must be either true or false')
self.response.out.write(
atom.activities_to_atom(activities,
actor,
host_url=url
or self.request.host_url + '/',
request_url=self.request.url,
xml_base=util.base_url(url),
title=title,
rels={'hub': hub} if hub else None,
reader=(reader == 'true')))
self.response.headers.add(
'Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format in ('as1-xml', 'xml'):
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
elif format == 'html':
self.response.headers['Content-Type'] = 'text/html'
self.response.out.write(
microformats2.activities_to_html(activities))
elif format in ('mf2-json', 'json-mf2'):
self.response.headers['Content-Type'] = 'application/json'
items = [microformats2.activity_to_json(a) for a in activities]
self.response.out.write(json.dumps({'items': items}, indent=2))
elif format == 'jsonfeed':
self.response.headers['Content-Type'] = 'application/json'
try:
jf = jsonfeed.activities_to_jsonfeed(activities,
actor=actor,
title=title,
feed_url=self.request.url)
except TypeError as e:
raise exc.HTTPBadRequest('Unsupported input data: %s' % e)
self.response.out.write(json.dumps(jf, indent=2))
if 'plaintext' in self.request.params:
# override response content type
self.response.headers['Content-Type'] = 'text/plain'
def test_from_as1_blank(self):
self.assertEqual({}, as2.from_as1(None))
self.assertEqual({}, as2.from_as1({}))
