def send_webmentions(handler, activity, **response_props): """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery. Args: handler: RequestHandler activity: dict, AS1 activity response_props: passed through to the newly created Responses """ verb = activity.get('verb') if verb and verb not in SUPPORTED_VERBS: error(handler, '%s activities are not supported yet.' % verb) # extract source and targets source = activity.get('url') or activity.get('id') obj = activity.get('object') obj_url = util.get_url(obj) targets = util.get_list(activity, 'inReplyTo') if isinstance(obj, dict): if not source: source = obj_url or obj.get('id') targets.extend(util.get_list(obj, 'inReplyTo')) if verb in ('like', 'share'): targets.append(obj_url) targets = util.dedupe_urls(util.get_url(t) for t in targets) if not source: error(handler, "Couldn't find original post URL") if not targets: error(handler, "Couldn't find target URLs (inReplyTo or object)") # send webmentions and store Responses errors = [] for target in targets: if not target: continue response = Response(source=source, target=target, direction='in', **response_props) response.put() wm_source = response.proxy_url() if verb in ('like', 'share') else source logging.info('Sending webmention from %s to %s', wm_source, target) wm = send.WebmentionSend(wm_source, target) if wm.send(headers=HEADERS): logging.info('Success: %s', wm.response) response.status = 'complete' else: logging.warning('Failed: %s', wm.error) errors.append(wm.error) response.status = 'error' response.put() if errors: msg = 'Errors:\n' + '\n'.join(json.dumps(e, indent=2) for e in errors) error(handler, msg, status=errors[0].get('http_status'))
def test_webmention_tools_relative_webmention_endpoint_in_header(self): requests.get('http://target/', verify=False).AndReturn( requests_response( '', headers={'Link': '</endpoint>; rel="webmention"'})) self.mox.ReplayAll() mention = send.WebmentionSend('http://source/', 'http://target/') mention.requests_kwargs = {} mention._discoverEndpoint() self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_header(self): super(testutil.HandlerTest, self).expect_requests_get( 'http://target/', '', verify=False, response_headers={'Link': '</endpoint>; rel="webmention"'}) self.mox.ReplayAll() mention = send.WebmentionSend('http://source/', 'http://target/') mention.requests_kwargs = {'timeout': HTTP_TIMEOUT} mention._discoverEndpoint() self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_body(self): requests.get('http://target/', verify=False).AndReturn( requests_response(""" <html><meta> <link rel="webmention" href="/endpoint"> </meta></html>""")) self.mox.ReplayAll() mention = send.WebmentionSend('http://source/', 'http://target/') mention.requests_kwargs = {} mention._discoverEndpoint() self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_body(self): super(testutil.HandlerTest, self).expect_requests_get('http://target/', """ <html><meta> <link rel="webmention" href="/endpoint"> </meta></html>""", verify=False) self.mox.ReplayAll() mention = send.WebmentionSend('http://source/', 'http://target/') mention.requests_kwargs = {'timeout': HTTP_TIMEOUT} mention._discoverEndpoint() self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def verify(self, force=False): """Checks that this source is ready to be used. For blog and listen sources, this fetches their front page HTML and discovers their webmention endpoint. For publish sources, this checks that they have a domain. May be overridden by subclasses, e.g. :class:`tumblr.Tumblr`. Args: force: if True, fully verifies (e.g. re-fetches the blog's HTML and performs webmention discovery) even we already think this source is verified. """ author_urls = [ u for u, d in zip(self.get_author_urls(), self.domains) if not util.in_webmention_blacklist(d) ] if ((self.verified() and not force) or self.status == 'disabled' or not self.features or not author_urls): return author_url = author_urls[0] logging.info('Attempting to discover webmention endpoint on %s', author_url) mention = send.WebmentionSend('https://brid.gy/', author_url) mention.requests_kwargs = { 'timeout': HTTP_TIMEOUT, 'headers': util.REQUEST_HEADERS } try: mention._discoverEndpoint() except BaseException: logging.info('Error discovering webmention endpoint', exc_info=True) mention.error = {'code': 'EXCEPTION'} self._fetched_html = getattr(mention, 'html', None) error = getattr(mention, 'error', None) endpoint = getattr(mention, 'receiver_endpoint', None) if error or not endpoint: logging.info("No webmention endpoint found: %s %r", error, endpoint) self.webmention_endpoint = None else: logging.info("Discovered webmention endpoint %s", endpoint) self.webmention_endpoint = endpoint self.put()
def do_send_webmentions(self): urls = self.entity.unsent + self.entity.error + self.entity.failed unsent = set() self.entity.error = [] self.entity.failed = [] for orig_url in urls: # recheck the url here since the checks may have failed during the poll # or streaming add. url, domain, ok = util.get_webmention_target(orig_url) if ok: if len(url) <= _MAX_STRING_LENGTH: unsent.add(url) else: logging.info('Giving up on target URL over %s chars! %s', _MAX_STRING_LENGTH, url) self.entity.failed.append(orig_url) self.entity.unsent = sorted(unsent) while self.entity.unsent: target = self.entity.unsent.pop(0) source_url = self.source_url(target) logging.info('Webmention from %s to %s', source_url, target) # see if we've cached webmention discovery for this domain. the cache # value is a string URL endpoint if discovery succeeded, a # WebmentionSend error dict if it failed (semi-)permanently, or None. cache_key = util.webmention_endpoint_cache_key(target) cached = util.webmention_endpoint_cache.get(cache_key) if cached: logging.info('Using cached webmention endpoint %r: %s', cache_key, cached) # send! and handle response or error error = None if isinstance(cached, dict): error = cached else: mention = send.WebmentionSend(source_url, target, endpoint=cached) headers = util.request_headers(source=self.source) logging.info('Sending...') try: if not mention.send(timeout=999, headers=headers): error = mention.error except BaseException as e: logging.info('', stack_info=True) error = getattr(mention, 'error') if not error: error = ({ 'code': 'BAD_TARGET_URL', 'http_status': 499 } if 'DNS lookup failed for URL:' in str(e) else { 'code': 'EXCEPTION' }) error_code = error['code'] if error else None if error_code != 'BAD_TARGET_URL' and not cached: val = error if error_code == 'NO_ENDPOINT' else mention.receiver_endpoint with util.webmention_endpoint_cache_lock: util.webmention_endpoint_cache[cache_key] = val if error is None: logging.info('Sent! %s', mention.response) self.record_source_webmention(mention) self.entity.sent.append(target) else: status = error.get('http_status', 0) if (error_code == 'NO_ENDPOINT' or (error_code == 'BAD_TARGET_URL' and status == 204)): # No Content logging.info('Giving up this target. %s', error) self.entity.skipped.append(target) elif status // 100 == 4: # Give up on 4XX errors; we don't expect later retries to succeed. logging.info('Giving up this target. %s', error) self.entity.failed.append(target) else: self.fail('Error sending to endpoint: %s' % error, level=logging.INFO) self.entity.error.append(target) if target in self.entity.unsent: self.entity.unsent.remove(target) if self.entity.error: logging.info('Propagate task failed') self.release('error') else: self.complete()
def send_webmentions(handler, activity_wrapped, proxy=None, **response_props): """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery. Args: handler: RequestHandler activity_wrapped: dict, AS1 activity response_props: passed through to the newly created Responses """ activity = common.redirect_unwrap(activity_wrapped) verb = activity.get('verb') if verb and verb not in SUPPORTED_VERBS: error(handler, '%s activities are not supported yet.' % verb) # extract source and targets source = activity.get('url') or activity.get('id') obj = activity.get('object') obj_url = util.get_url(obj) targets = util.get_list(activity, 'inReplyTo') if isinstance(obj, dict): if not source or verb in ('create', 'post', 'update'): source = obj_url or obj.get('id') targets.extend(util.get_list(obj, 'inReplyTo')) tags = util.get_list(activity_wrapped, 'tags') obj_wrapped = activity_wrapped.get('object') if isinstance(obj_wrapped, dict): tags.extend(util.get_list(obj_wrapped, 'tags')) for tag in tags: if tag.get('objectType') == 'mention': url = tag.get('url') if url and url.startswith(appengine_config.HOST_URL): targets.append(redirect_unwrap(url)) if verb in ('follow', 'like', 'share'): targets.append(obj_url) targets = util.dedupe_urls(util.get_url(t) for t in targets) if not source: error(handler, "Couldn't find original post URL") if not targets: error(handler, "Couldn't find any target URLs in inReplyTo, object, or mention tags") # send webmentions and store Responses errors = [] for target in targets: if util.domain_from_link(target) == util.domain_from_link(source): logging.info('Skipping same-domain webmention from %s to %s', source, target) continue response = Response(source=source, target=target, direction='in', **response_props) response.put() wm_source = (response.proxy_url() if verb in ('follow', 'like', 'share') or proxy else source) logging.info('Sending webmention from %s to %s', wm_source, target) wm = send.WebmentionSend(wm_source, target) if wm.send(headers=HEADERS): logging.info('Success: %s', wm.response) response.status = 'complete' else: logging.warning('Failed: %s', wm.error) errors.append(wm.error) response.status = 'error' response.put() if errors: msg = 'Errors:\n' + '\n'.join(json.dumps(e, indent=2) for e in errors) error(handler, msg, status=errors[0].get('http_status'))