def send_webmentions(handler, activity, **response_props):
"""Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery.
Args:
handler: RequestHandler
activity: dict, AS1 activity
response_props: passed through to the newly created Responses
"""
verb = activity.get('verb')
if verb and verb not in SUPPORTED_VERBS:
error(handler, '%s activities are not supported yet.' % verb)
# extract source and targets
source = activity.get('url') or activity.get('id')
obj = activity.get('object')
obj_url = util.get_url(obj)
targets = util.get_list(activity, 'inReplyTo')
if isinstance(obj, dict):
if not source:
source = obj_url or obj.get('id')
targets.extend(util.get_list(obj, 'inReplyTo'))
if verb in ('like', 'share'):
targets.append(obj_url)
targets = util.dedupe_urls(util.get_url(t) for t in targets)
if not source:
error(handler, "Couldn't find original post URL")
if not targets:
error(handler, "Couldn't find target URLs (inReplyTo or object)")
# send webmentions and store Responses
errors = []
for target in targets:
if not target:
continue
response = Response(source=source, target=target, direction='in',
**response_props)
response.put()
wm_source = response.proxy_url() if verb in ('like', 'share') else source
logging.info('Sending webmention from %s to %s', wm_source, target)
wm = send.WebmentionSend(wm_source, target)
if wm.send(headers=HEADERS):
logging.info('Success: %s', wm.response)
response.status = 'complete'
else:
logging.warning('Failed: %s', wm.error)
errors.append(wm.error)
response.status = 'error'
response.put()
if errors:
msg = 'Errors:\n' + '\n'.join(json.dumps(e, indent=2) for e in errors)
error(handler, msg, status=errors[0].get('http_status'))
def test_webmention_tools_relative_webmention_endpoint_in_header(self):
requests.get('http://target/', verify=False).AndReturn(
requests_response(
'', headers={'Link': '</endpoint>; rel="webmention"'}))
self.mox.ReplayAll()
mention = send.WebmentionSend('http://source/', 'http://target/')
mention.requests_kwargs = {}
mention._discoverEndpoint()
self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_header(self):
super(testutil.HandlerTest, self).expect_requests_get(
'http://target/',
'',
verify=False,
response_headers={'Link': '</endpoint>; rel="webmention"'})
self.mox.ReplayAll()
mention = send.WebmentionSend('http://source/', 'http://target/')
mention.requests_kwargs = {'timeout': HTTP_TIMEOUT}
mention._discoverEndpoint()
self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_body(self):
requests.get('http://target/', verify=False).AndReturn(
requests_response("""
<html><meta>
<link rel="webmention" href="/endpoint">
</meta></html>"""))
self.mox.ReplayAll()
mention = send.WebmentionSend('http://source/', 'http://target/')
mention.requests_kwargs = {}
mention._discoverEndpoint()
self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def test_webmention_tools_relative_webmention_endpoint_in_body(self):
super(testutil.HandlerTest, self).expect_requests_get('http://target/',
"""
<html><meta>
<link rel="webmention" href="/endpoint">
</meta></html>""",
verify=False)
self.mox.ReplayAll()
mention = send.WebmentionSend('http://source/', 'http://target/')
mention.requests_kwargs = {'timeout': HTTP_TIMEOUT}
mention._discoverEndpoint()
self.assertEquals('http://target/endpoint', mention.receiver_endpoint)
def verify(self, force=False):
"""Checks that this source is ready to be used.
For blog and listen sources, this fetches their front page HTML and
discovers their webmention endpoint. For publish sources, this checks that
they have a domain.
May be overridden by subclasses, e.g. :class:`tumblr.Tumblr`.
Args:
force: if True, fully verifies (e.g. re-fetches the blog's HTML and
performs webmention discovery) even we already think this source is
verified.
"""
author_urls = [
u for u, d in zip(self.get_author_urls(), self.domains)
if not util.in_webmention_blacklist(d)
]
if ((self.verified() and not force) or self.status == 'disabled'
or not self.features or not author_urls):
return
author_url = author_urls[0]
logging.info('Attempting to discover webmention endpoint on %s',
author_url)
mention = send.WebmentionSend('https://brid.gy/', author_url)
mention.requests_kwargs = {
'timeout': HTTP_TIMEOUT,
'headers': util.REQUEST_HEADERS
}
try:
mention._discoverEndpoint()
except BaseException:
logging.info('Error discovering webmention endpoint',
exc_info=True)
mention.error = {'code': 'EXCEPTION'}
self._fetched_html = getattr(mention, 'html', None)
error = getattr(mention, 'error', None)
endpoint = getattr(mention, 'receiver_endpoint', None)
if error or not endpoint:
logging.info("No webmention endpoint found: %s %r", error,
endpoint)
self.webmention_endpoint = None
else:
logging.info("Discovered webmention endpoint %s", endpoint)
self.webmention_endpoint = endpoint
self.put()
def do_send_webmentions(self):
urls = self.entity.unsent + self.entity.error + self.entity.failed
unsent = set()
self.entity.error = []
self.entity.failed = []
for orig_url in urls:
# recheck the url here since the checks may have failed during the poll
# or streaming add.
url, domain, ok = util.get_webmention_target(orig_url)
if ok:
if len(url) <= _MAX_STRING_LENGTH:
unsent.add(url)
else:
logging.info('Giving up on target URL over %s chars! %s',
_MAX_STRING_LENGTH, url)
self.entity.failed.append(orig_url)
self.entity.unsent = sorted(unsent)
while self.entity.unsent:
target = self.entity.unsent.pop(0)
source_url = self.source_url(target)
logging.info('Webmention from %s to %s', source_url, target)
# see if we've cached webmention discovery for this domain. the cache
# value is a string URL endpoint if discovery succeeded, a
# WebmentionSend error dict if it failed (semi-)permanently, or None.
cache_key = util.webmention_endpoint_cache_key(target)
cached = util.webmention_endpoint_cache.get(cache_key)
if cached:
logging.info('Using cached webmention endpoint %r: %s',
cache_key, cached)
# send! and handle response or error
error = None
if isinstance(cached, dict):
error = cached
else:
mention = send.WebmentionSend(source_url,
target,
endpoint=cached)
headers = util.request_headers(source=self.source)
logging.info('Sending...')
try:
if not mention.send(timeout=999, headers=headers):
error = mention.error
except BaseException as e:
logging.info('', stack_info=True)
error = getattr(mention, 'error')
if not error:
error = ({
'code': 'BAD_TARGET_URL',
'http_status': 499
} if 'DNS lookup failed for URL:' in str(e) else {
'code': 'EXCEPTION'
})
error_code = error['code'] if error else None
if error_code != 'BAD_TARGET_URL' and not cached:
val = error if error_code == 'NO_ENDPOINT' else mention.receiver_endpoint
with util.webmention_endpoint_cache_lock:
util.webmention_endpoint_cache[cache_key] = val
if error is None:
logging.info('Sent! %s', mention.response)
self.record_source_webmention(mention)
self.entity.sent.append(target)
else:
status = error.get('http_status', 0)
if (error_code == 'NO_ENDPOINT'
or (error_code == 'BAD_TARGET_URL'
and status == 204)): # No Content
logging.info('Giving up this target. %s', error)
self.entity.skipped.append(target)
elif status // 100 == 4:
# Give up on 4XX errors; we don't expect later retries to succeed.
logging.info('Giving up this target. %s', error)
self.entity.failed.append(target)
else:
self.fail('Error sending to endpoint: %s' % error,
level=logging.INFO)
self.entity.error.append(target)
if target in self.entity.unsent:
self.entity.unsent.remove(target)
if self.entity.error:
logging.info('Propagate task failed')
self.release('error')
else:
self.complete()
def send_webmentions(handler, activity_wrapped, proxy=None, **response_props):
"""Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery.
Args:
handler: RequestHandler
activity_wrapped: dict, AS1 activity
response_props: passed through to the newly created Responses
"""
activity = common.redirect_unwrap(activity_wrapped)
verb = activity.get('verb')
if verb and verb not in SUPPORTED_VERBS:
error(handler, '%s activities are not supported yet.' % verb)
# extract source and targets
source = activity.get('url') or activity.get('id')
obj = activity.get('object')
obj_url = util.get_url(obj)
targets = util.get_list(activity, 'inReplyTo')
if isinstance(obj, dict):
if not source or verb in ('create', 'post', 'update'):
source = obj_url or obj.get('id')
targets.extend(util.get_list(obj, 'inReplyTo'))
tags = util.get_list(activity_wrapped, 'tags')
obj_wrapped = activity_wrapped.get('object')
if isinstance(obj_wrapped, dict):
tags.extend(util.get_list(obj_wrapped, 'tags'))
for tag in tags:
if tag.get('objectType') == 'mention':
url = tag.get('url')
if url and url.startswith(appengine_config.HOST_URL):
targets.append(redirect_unwrap(url))
if verb in ('follow', 'like', 'share'):
targets.append(obj_url)
targets = util.dedupe_urls(util.get_url(t) for t in targets)
if not source:
error(handler, "Couldn't find original post URL")
if not targets:
error(handler, "Couldn't find any target URLs in inReplyTo, object, or mention tags")
# send webmentions and store Responses
errors = []
for target in targets:
if util.domain_from_link(target) == util.domain_from_link(source):
logging.info('Skipping same-domain webmention from %s to %s',
source, target)
continue
response = Response(source=source, target=target, direction='in',
**response_props)
response.put()
wm_source = (response.proxy_url()
if verb in ('follow', 'like', 'share') or proxy
else source)
logging.info('Sending webmention from %s to %s', wm_source, target)
wm = send.WebmentionSend(wm_source, target)
if wm.send(headers=HEADERS):
logging.info('Success: %s', wm.response)
response.status = 'complete'
else:
logging.warning('Failed: %s', wm.error)
errors.append(wm.error)
response.status = 'error'
response.put()
if errors:
msg = 'Errors:\n' + '\n'.join(json.dumps(e, indent=2) for e in errors)
error(handler, msg, status=errors[0].get('http_status'))
