def test_bad_input_types(self): for bad in 1, [2], (3, ): for fn in as2.to_as1, as2.from_as1: with self.assertRaises(ValueError): fn(bad) with self.assertRaises(ValueError): as2.from_as1('z')
def test_bad_input_types(self): for bad in 1, [2], (3,): for fn in as2.to_as1, as2.from_as1: with self.assertRaises(ValueError): fn(bad) with self.assertRaises(ValueError): as2.from_as1('z')
def get(self, domain): url = 'http://%s/' % domain resp = common.requests_get(url) mf2 = mf2py.parse(resp.text, url=resp.url, img_with_alt=True) # logging.info('Parsed mf2 for %s: %s', resp.url, json.dumps(mf2, indent=2)) hcard = mf2util.representative_hcard(mf2, resp.url) logging.info('Representative h-card: %s', json.dumps(hcard, indent=2)) if not hcard: common.error( self, """\ Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % resp.url) key = MagicKey.get_or_create(domain) obj = common.postprocess_as2(as2.from_as1( microformats2.json_to_object(hcard)), key=key) obj.update({ 'inbox': '%s/%s/inbox' % (appengine_config.HOST_URL, domain), 'outbox': '%s/%s/outbox' % (appengine_config.HOST_URL, domain), 'following': '%s/%s/following' % (appengine_config.HOST_URL, domain), 'followers': '%s/%s/followers' % (appengine_config.HOST_URL, domain), }) logging.info('Returning: %s', json.dumps(obj, indent=2)) self.response.headers.update({ 'Content-Type': common.CONTENT_TYPE_AS2, 'Access-Control-Allow-Origin': '*', }) self.response.write(json.dumps(obj, indent=2))
def actor(domain): """Serves /[DOMAIN], fetches its mf2, converts to AS Actor, and serves it.""" tld = domain.split('.')[-1] if tld in common.TLD_BLOCKLIST: error('', status=404) mf2 = util.fetch_mf2(f'http://{domain}/', gateway=True, headers=common.HEADERS) hcard = mf2util.representative_hcard(mf2, mf2['url']) logging.info(f'Representative h-card: {json_dumps(hcard, indent=2)}') if not hcard: error( f"Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on {mf2['url']}" ) key = MagicKey.get_or_create(domain) obj = common.postprocess_as2(as2.from_as1( microformats2.json_to_object(hcard)), key=key) obj.update({ 'preferredUsername': domain, 'inbox': f'{request.host_url}{domain}/inbox', 'outbox': f'{request.host_url}{domain}/outbox', 'following': f'{request.host_url}{domain}/following', 'followers': f'{request.host_url}{domain}/followers', }) logging.info(f'Returning: {json_dumps(obj, indent=2)}') return (obj, { 'Content-Type': common.CONTENT_TYPE_AS2, 'Access-Control-Allow-Origin': '*', })
def get(self, domain): tld = domain.split('.')[-1] if tld in common.TLD_BLOCKLIST: self.error('', status=404) mf2 = util.fetch_mf2('http://%s/' % domain, gateway=True, headers=common.HEADERS) # logging.info('Parsed mf2 for %s: %s', resp.url, json_dumps(mf2, indent=2)) hcard = mf2util.representative_hcard(mf2, mf2['url']) logging.info('Representative h-card: %s', json_dumps(hcard, indent=2)) if not hcard: self.error("""\ Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % mf2['url']) key = MagicKey.get_or_create(domain) obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)), key=key) obj.update({ 'inbox': '%s/%s/inbox' % (self.request.host_url, domain), 'outbox': '%s/%s/outbox' % (self.request.host_url, domain), 'following': '%s/%s/following' % (self.request.host_url, domain), 'followers': '%s/%s/followers' % (self.request.host_url, domain), }) logging.info('Returning: %s', json_dumps(obj, indent=2)) self.response.headers.update({ 'Content-Type': common.CONTENT_TYPE_AS2, 'Access-Control-Allow-Origin': '*', }) self.response.write(json_dumps(obj, indent=2))
def try_activitypub(self): """Returns True if we attempted ActivityPub delivery, False otherwise.""" targets = self._activitypub_targets() if not targets: return False key = MagicKey.get_or_create(self.source_domain) error = None last_success = None # TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients for resp, inbox in targets: target_obj = json_loads( resp.target_as2) if resp.target_as2 else None source_activity = self.postprocess_as2(as2.from_as1( self.source_obj), target=target_obj, key=key) if resp.status == 'complete': source_activity['type'] = 'Update' try: last = activitypub.send(source_activity, inbox, self.source_domain) resp.status = 'complete' last_success = last except BaseException as e: error = e resp.status = 'error' resp.put() # Pass the AP response status code and body through as our response if last_success: self.response.status_int = last_success.status_code self.response.write(last_success.text) elif isinstance(error, requests.HTTPError): self.response.status_int = error.status_code self.response.write(error.text) else: self.response.write(str(error)) return bool(last_success)
def convert_to_as2(url): """Fetch a URL as HTML, convert it to AS2, and return it. Currently mainly for Pixelfed. https://github.com/snarfed/bridgy-fed/issues/39 """ mf2 = util.fetch_mf2(url) entry = mf2util.find_first_entry(mf2, ['h-entry']) logging.info(f"Parsed mf2 for {mf2['url']}: {json_dumps(entry, indent=2)}") obj = common.postprocess_as2( as2.from_as1(microformats2.json_to_object(entry))) logging.info(f'Returning: {json_dumps(obj, indent=2)}') return obj, { 'Content-Type': common.CONTENT_TYPE_AS2, 'Access-Control-Allow-Origin': '*', }
def convert_to_as2(self, url): """Fetch a URL as HTML, convert it to AS2, and return it. Currently mainly for Pixelfed. https://github.com/snarfed/bridgy-fed/issues/39 """ mf2 = util.fetch_mf2(url) entry = mf2util.find_first_entry(mf2, ['h-entry']) logging.info('Parsed mf2 for %s: %s', mf2['url'], json_dumps(entry, indent=2)) obj = self.postprocess_as2( as2.from_as1(microformats2.json_to_object(entry))) logging.info('Returning: %s', json_dumps(obj, indent=2)) self.response.headers.update({ 'Content-Type': common.CONTENT_TYPE_AS2, 'Access-Control-Allow-Origin': '*', }) self.response.write(json_dumps(obj, indent=2))
def test_from_as1_blank(self): self.assertEqual({}, as2.from_as1(None)) self.assertEqual({}, as2.from_as1({}))
def write_response(self, response, actor=None, url=None, title=None, hfeed=None): """Converts ActivityStreams activities and writes them out. Args: response: response dict with values based on OpenSocial ActivityStreams REST API, as returned by Source.get_activities_response() actor: optional ActivityStreams actor dict for current user. Only used for Atom and JSON Feed output. url: the input URL title: string, used in feed output (Atom, JSON Feed, RSS) hfeed: dict, parsed mf2 h-feed, if available """ format = self.request.get('format') or self.request.get( 'output') or 'json' if format not in FORMATS: raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' % (format, FORMATS)) if 'plaintext' in self.request.params: # override content type self.response.headers['Content-Type'] = 'text/plain' else: content_type = FORMATS.get(format) if content_type: self.response.headers['Content-Type'] = content_type if self.request.method == 'HEAD': return activities = response['items'] try: if format in ('as1', 'json', 'activitystreams'): self.response.out.write(json_dumps(response, indent=2)) elif format == 'as2': response.update({ 'items': [as2.from_as1(a) for a in activities], 'totalItems': response.pop('totalResults', None), 'updated': response.pop('updatedSince', None), 'filtered': None, 'sorted': None, }) self.response.out.write( json_dumps(util.trim_nulls(response), indent=2)) elif format == 'atom': hub = self.request.get('hub') reader = self.request.get('reader', 'true').lower() if reader not in ('true', 'false'): self.abort(400, 'reader param must be either true or false') if not actor and hfeed: actor = microformats2.json_to_object({ 'properties': hfeed.get('properties', {}), }) self.response.out.write( atom.activities_to_atom(activities, actor, host_url=url or self.request.host_url + '/', request_url=self.request.url, xml_base=util.base_url(url), title=title, rels={'hub': hub} if hub else None, reader=(reader == 'true'))) self.response.headers.add( 'Link', str('<%s>; rel="self"' % self.request.url)) if hub: self.response.headers.add('Link', str('<%s>; rel="hub"' % hub)) elif format == 'rss': if not title: title = 'Feed for %s' % url self.response.out.write( rss.from_activities(activities, actor, title=title, feed_url=self.request.url, hfeed=hfeed, home_page_url=util.base_url(url))) elif format in ('as1-xml', 'xml'): self.response.out.write(XML_TEMPLATE % util.to_xml(response)) elif format == 'html': self.response.out.write( microformats2.activities_to_html(activities)) elif format in ('mf2-json', 'json-mf2'): items = [microformats2.activity_to_json(a) for a in activities] self.response.out.write(json_dumps({'items': items}, indent=2)) elif format == 'jsonfeed': try: jf = jsonfeed.activities_to_jsonfeed( activities, actor=actor, title=title, feed_url=self.request.url) except TypeError as e: raise exc.HTTPBadRequest('Unsupported input data: %s' % e) self.response.out.write(json_dumps(jf, indent=2)) except ValueError as e: logging.warning('converting to output format failed', stack_info=True) self.abort(400, 'Could not convert to %s: %s' % (format, str(e)))
def try_activitypub(self): source = util.get_required_param(self, 'source') # fetch source page, convert to ActivityStreams source_resp = common.requests_get(source) source_url = source_resp.url or source source_mf2 = mf2py.parse(source_resp.text, url=source_url) # logging.debug('Parsed mf2 for %s: %s', source_resp.url, json.dumps(source_mf2, indent=2)) entry = mf2util.find_first_entry(source_mf2, ['h-entry']) logging.info('First entry: %s', json.dumps(entry, indent=2)) # make sure it has url, since we use that for AS2 id, which is required # for ActivityPub. props = entry.setdefault('properties', {}) if not props.get('url'): props['url'] = [source_url] source_obj = microformats2.json_to_object(entry, fetch_mf2=True) logging.info('Converted to AS: %s', json.dumps(source_obj, indent=2)) # fetch target page as AS object. target is first in-reply-to, like-of, # or repost-of, *not* target query param.) target = util.get_url(util.get_first(source_obj, 'inReplyTo') or util.get_first(source_obj, 'object')) if not target: common.error(self, 'No u-in-reply-to, u-like-of, or u-repost-of ' 'found in %s' % source_url) try: target_resp = common.get_as2(target) except (requests.HTTPError, exc.HTTPBadGateway) as e: if (e.response.status_code // 100 == 2 and common.content_type(e.response).startswith('text/html')): self.resp = Response.get_or_create( source=source_url, target=e.response.url or target, direction='out', source_mf2=json.dumps(source_mf2)) return self.send_salmon(source_obj, target_resp=e.response) raise target_url = target_resp.url or target self.resp = Response.get_or_create( source=source_url, target=target_url, direction='out', protocol='activitypub', source_mf2=json.dumps(source_mf2)) # find actor's inbox target_obj = target_resp.json() inbox_url = target_obj.get('inbox') if not inbox_url: # TODO: test actor/attributedTo and not, with/without inbox actor = target_obj.get('actor') or target_obj.get('attributedTo') if isinstance(actor, dict): inbox_url = actor.get('inbox') actor = actor.get('url') if not inbox_url and not actor: common.error(self, 'Target object has no actor or attributedTo URL') if not inbox_url: # fetch actor as AS object actor = common.get_as2(actor).json() inbox_url = actor.get('inbox') if not inbox_url: # TODO: probably need a way to save errors like this so that we can # return them if ostatus fails too. # common.error(self, 'Target actor has no inbox') return self.send_salmon(source_obj, target_resp=target_resp) # convert to AS2 source_domain = urlparse.urlparse(source_url).netloc key = MagicKey.get_or_create(source_domain) source_activity = common.postprocess_as2( as2.from_as1(source_obj), target=target_obj, key=key) if self.resp.status == 'complete': source_activity['type'] = 'Update' # prepare HTTP Signature (required by Mastodon) # https://w3c.github.io/activitypub/#authorization-lds # https://tools.ietf.org/html/draft-cavage-http-signatures-07 # https://github.com/tootsuite/mastodon/issues/4906#issuecomment-328844846 acct = 'acct:%s@%s' % (source_domain, source_domain) auth = HTTPSignatureAuth(secret=key.private_pem(), key_id=acct, algorithm='rsa-sha256') # deliver source object to target actor's inbox. headers = { 'Content-Type': common.CONTENT_TYPE_AS2, # required for HTTP Signature # https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3 'Date': datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'), } inbox_url = urlparse.urljoin(target_url, inbox_url) resp = common.requests_post(inbox_url, json=source_activity, auth=auth, headers=headers) self.response.status_int = resp.status_code if resp.status_code == 202: self.response.write('202 response! If this is Mastodon 1.x, their ' 'signature verification probably failed. :(\n') self.response.write(resp.text)
def try_activitypub(self): """Attempts ActivityPub delivery. Returns Flask response (string body or tuple) if we succeeded or failed, None if ActivityPub was not available. """ targets = self._activitypub_targets() if not targets: return None key = MagicKey.get_or_create(self.source_domain) error = None last_success = None # TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients for resp, inbox in targets: target_obj = json_loads( resp.target_as2) if resp.target_as2 else None source_activity = common.postprocess_as2(as2.from_as1( self.source_obj), target=target_obj, key=key) if resp.status == 'complete': if resp.source_mf2: def content(mf2): items = mf2.get('items') if items: return microformats2.first_props( items[0].get('properties')).get('content') orig_content = content(json_loads(resp.source_mf2)) new_content = content(self.source_mf2) if orig_content and new_content and orig_content == new_content: msg = f'Skipping; new content is same as content published before at {resp.updated}' logging.info(msg) return msg source_activity['type'] = 'Update' try: last = activitypub.send(source_activity, inbox, self.source_domain) resp.status = 'complete' last_success = last except BaseException as e: error = e resp.status = 'error' resp.put() # Pass the AP response status code and body through as our response if last_success: return last_success.text or 'Sent!', last_success.status_code elif isinstance(error, BadGateway): raise error elif isinstance(error, requests.HTTPError): return str(error), error.status_code else: return str(error)
def write_response(self, response, actor=None, url=None, title=None, hfeed=None): """Converts ActivityStreams activities and writes them out. Args: response: response dict with values based on OpenSocial ActivityStreams REST API, as returned by Source.get_activities_response() actor: optional ActivityStreams actor dict for current user. Only used for Atom and JSON Feed output. url: the input URL title: string, used in feed output (Atom, JSON Feed, RSS) hfeed: dict, parsed mf2 h-feed, if available """ format = self.request.get('format') or self.request.get('output') or 'json' if format not in FORMATS: raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' % (format, FORMATS)) activities = response['items'] try: if format in ('as1', 'json', 'activitystreams'): # list of official MIME types: # https://www.iana.org/assignments/media-types/media-types.xhtml self.response.headers['Content-Type'] = \ 'application/json' if format == 'json' else 'application/stream+json' self.response.out.write(json.dumps(response, indent=2)) elif format == 'as2': self.response.headers['Content-Type'] = 'application/activity+json' response.update({ 'items': [as2.from_as1(a) for a in activities], 'totalItems': response.pop('totalResults', None), 'updated': response.pop('updatedSince', None), 'filtered': None, 'sorted': None, }) self.response.out.write(json.dumps(util.trim_nulls(response), indent=2)) elif format == 'atom': self.response.headers['Content-Type'] = 'application/atom+xml' hub = self.request.get('hub') reader = self.request.get('reader', 'true').lower() if reader not in ('true', 'false'): self.abort(400, 'reader param must be either true or false') self.response.out.write(atom.activities_to_atom( activities, actor, host_url=url or self.request.host_url + '/', request_url=self.request.url, xml_base=util.base_url(url), title=title, rels={'hub': hub} if hub else None, reader=(reader == 'true'))) self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url)) if hub: self.response.headers.add('Link', str('<%s>; rel="hub"' % hub)) elif format == 'rss': self.response.headers['Content-Type'] = 'application/rss+xml' if not title: title = 'Feed for %s' % url self.response.out.write(rss.from_activities( activities, actor, title=title, feed_url=self.request.url, hfeed=hfeed, home_page_url=util.base_url(url))) elif format in ('as1-xml', 'xml'): self.response.headers['Content-Type'] = 'application/xml' self.response.out.write(XML_TEMPLATE % util.to_xml(response)) elif format == 'html': self.response.headers['Content-Type'] = 'text/html' self.response.out.write(microformats2.activities_to_html(activities)) elif format in ('mf2-json', 'json-mf2'): self.response.headers['Content-Type'] = 'application/json' items = [microformats2.activity_to_json(a) for a in activities] self.response.out.write(json.dumps({'items': items}, indent=2)) elif format == 'jsonfeed': self.response.headers['Content-Type'] = 'application/json' try: jf = jsonfeed.activities_to_jsonfeed(activities, actor=actor, title=title, feed_url=self.request.url) except TypeError as e: raise exc.HTTPBadRequest('Unsupported input data: %s' % e) self.response.out.write(json.dumps(jf, indent=2)) except ValueError as e: logging.warning('converting to output format failed', exc_info=True) self.abort(400, 'Could not convert to %s: %s' % (format, str(e))) if 'plaintext' in self.request.params: # override response content type self.response.headers['Content-Type'] = 'text/plain'
def write_response(self, response, actor=None, url=None, title=None): """Converts ActivityStreams activities and writes them out. Args: response: response dict with values based on OpenSocial ActivityStreams REST API, as returned by Source.get_activities_response() actor: optional ActivityStreams actor dict for current user. Only used for Atom and JSON Feed output. url: the input URL title: string, Used in Atom and JSON Feed output """ format = self.request.get('format') or self.request.get( 'output') or 'json' if format not in FORMATS: raise exc.HTTPBadRequest('Invalid format: %s, expected one of %r' % (format, FORMATS)) activities = response['items'] if format in ('as1', 'json', 'activitystreams'): # list of official MIME types: # https://www.iana.org/assignments/media-types/media-types.xhtml self.response.headers['Content-Type'] = \ 'application/json' if format == 'json' else 'application/stream+json' self.response.out.write(json.dumps(response, indent=2)) elif format == 'as2': self.response.headers['Content-Type'] = 'application/activity+json' response.update({ 'items': [as2.from_as1(a) for a in activities], 'totalItems': response.pop('totalResults', None), 'updated': response.pop('updatedSince', None), 'filtered': None, 'sorted': None, }) self.response.out.write( json.dumps(util.trim_nulls(response), indent=2)) elif format == 'atom': self.response.headers['Content-Type'] = 'application/atom+xml' hub = self.request.get('hub') reader = self.request.get('reader', 'true').lower() if reader not in ('true', 'false'): self.abort(400, 'reader param must be either true or false') self.response.out.write( atom.activities_to_atom(activities, actor, host_url=url or self.request.host_url + '/', request_url=self.request.url, xml_base=util.base_url(url), title=title, rels={'hub': hub} if hub else None, reader=(reader == 'true'))) self.response.headers.add( 'Link', str('<%s>; rel="self"' % self.request.url)) if hub: self.response.headers.add('Link', str('<%s>; rel="hub"' % hub)) elif format in ('as1-xml', 'xml'): self.response.headers['Content-Type'] = 'application/xml' self.response.out.write(XML_TEMPLATE % util.to_xml(response)) elif format == 'html': self.response.headers['Content-Type'] = 'text/html' self.response.out.write( microformats2.activities_to_html(activities)) elif format in ('mf2-json', 'json-mf2'): self.response.headers['Content-Type'] = 'application/json' items = [microformats2.activity_to_json(a) for a in activities] self.response.out.write(json.dumps({'items': items}, indent=2)) elif format == 'jsonfeed': self.response.headers['Content-Type'] = 'application/json' try: jf = jsonfeed.activities_to_jsonfeed(activities, actor=actor, title=title, feed_url=self.request.url) except TypeError as e: raise exc.HTTPBadRequest('Unsupported input data: %s' % e) self.response.out.write(json.dumps(jf, indent=2)) if 'plaintext' in self.request.params: # override response content type self.response.headers['Content-Type'] = 'text/plain'