tasks.py

# coding=utf-8
"""Task queue handlers.
"""
import datetime
import gc
import logging

from google.cloud import ndb
from google.cloud.ndb._datastore_types import _MAX_STRING_LENGTH
from granary.source import Source
from oauth_dropins.webutil import appengine_info, logs, webmention
from oauth_dropins.webutil import handlers as webutil_handlers
from oauth_dropins.webutil.appengine_config import ndb_client
from oauth_dropins.webutil.util import json_dumps, json_loads
from requests import HTTPError
import webapp2

import appengine_config
import cron
import models
from models import Response
import original_post_discovery
import util
# need to import model class definitions since poll creates and saves entities.
import blogger, facebook, flickr, github, instagram, mastodon, medium, reddit, tumblr, twitter, wordpress_rest

# Used as a sentinel value in the webmention endpoint cache
NO_ENDPOINT = 'NONE'


class Poll(webapp2.RequestHandler):
  """Task handler that fetches and processes new responses from a single source.

  Request parameters:

  * source_key: string key of source entity
  * last_polled: timestamp, YYYY-MM-DD-HH-MM-SS

  Inserts a propagate task for each response that hasn't been seen before.

  Steps:
  1: Fetch activities: posts by the user, links to the user's domain(s).
  2: Extract responses, store their activities.
  3: Filter out responses we've already seen, using Responses in the datastore.
  4: Store new responses and enqueue propagate tasks.
  5: Possibly refetch updated syndication urls.

  1-4 are in backfeed(); 5 is in poll().
  """
  RESTART_EXISTING_TASKS = False  # overridden in Discover
  TRANSIENT_ERROR_HTTP_CODES = ()

  handle_exception = util.background_handle_exception

  def _last_poll_url(self, source):
    return '%s/%s' % (util.host_url(self),
                      logs.url(source.last_poll_attempt, source.key))

  def post(self, *path_args):
    self.request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
    logging.debug('Params: %s', list(self.request.params.items()))

    key = self.request.params['source_key']
    source = self.source = ndb.Key(urlsafe=key).get()
    if not source or source.status == 'disabled' or 'listen' not in source.features:
      logging.error('Source not found or disabled. Dropping task.')
      return
    logging.info('Source: %s %s, %s', source.label(), source.key_id(),
                 source.bridgy_url(self))

    if source.AUTO_POLL:
      last_polled = self.request.params['last_polled']
      if last_polled != source.last_polled.strftime(util.POLL_TASK_DATETIME_FORMAT):
        logging.warning('duplicate poll task! deferring to the other task.')
        return

    logging.info('Last poll: %s', self._last_poll_url(source))

    # mark this source as polling
    source.updates = {
      'poll_status': 'polling',
      'last_poll_attempt': util.now_fn(),
      'rate_limited': False,
    }
    source = models.Source.put_updates(source)

    source.updates = {}
    try:
      self.poll(source)
    except Exception as e:
      source.updates['poll_status'] = 'error'
      code, body = util.interpret_http_exception(e)
      if code in source.DISABLE_HTTP_CODES or isinstance(e, models.DisableSource):
        # the user deauthorized the bridgy app, so disable this source.
        # let the task complete successfully so that it's not retried.
        logging.warning('Disabling source due to: %s' % e, stack_info=True)
        source.updates.update({
          'status': 'disabled',
          'poll_status': 'ok',
        })
        body = '%s\nLast poll: %s' % (source.bridgy_url(self),
                                      self._last_poll_url(source))
      elif code in source.RATE_LIMIT_HTTP_CODES:
        logging.info('Rate limited. Marking as error and finishing. %s', e)
        source.updates['rate_limited'] = True
      else:
        raise
    finally:
      source = models.Source.put_updates(source)

    if source.AUTO_POLL:
      util.add_poll_task(source)

    # feeble attempt to avoid hitting the instance memory limit
    source = None
    gc.collect()

  def poll(self, source):
    """Actually runs the poll.

    Stores property names and values to update in source.updates.
    """
    if source.last_activities_etag or source.last_activity_id:
      logging.debug('Using ETag %s, last activity id %s',
                    source.last_activities_etag, source.last_activity_id)

    #
    # Step 1: fetch activities:
    # * posts by the user
    # * search all posts for the user's domain URLs to find links
    #
    cache = util.CacheDict()
    if source.last_activities_cache_json:
      cache.update(json_loads(source.last_activities_cache_json))

    # search for links first so that the user's activities and responses
    # override them if they overlap
    links = source.search_for_links()

    # this user's own activities (and user mentions)
    resp = source.get_activities_response(
      fetch_replies=True, fetch_likes=True, fetch_shares=True,
      fetch_mentions=True, count=50, etag=source.last_activities_etag,
      min_id=source.last_activity_id, cache=cache)
    etag = resp.get('etag')  # used later
    user_activities = resp.get('items', [])

    # these map ids to AS objects.
    # backfeed all links as responses, but only include the user's own links as
    # activities, since their responses also get backfeed.
    responses = {a['id']: a for a in links}

    user_id = source.user_tag_id()
    links_by_user = [a for a in links
                     if a.get('object', {}).get('author', {}).get('id') == user_id]
    activities = {a['id']: a for a in links_by_user + user_activities}

    # extract silo activity ids, update last_activity_id
    silo_activity_ids = set()
    last_activity_id = source.last_activity_id
    for id, activity in activities.items():
      # maybe replace stored last activity id
      parsed = util.parse_tag_uri(id)
      if parsed:
        id = parsed[1]
      silo_activity_ids.add(id)
      try:
        # try numeric comparison first
        greater = int(id) > int(last_activity_id)
      except (TypeError, ValueError):
        greater = str(id) > str(last_activity_id)
      if greater:
        last_activity_id = id

    if last_activity_id and last_activity_id != source.last_activity_id:
      source.updates['last_activity_id'] = last_activity_id

    # trim cache to just the returned activity ids, so that it doesn't grow
    # without bound. (WARNING: depends on get_activities_response()'s cache key
    # format, e.g. 'PREFIX ACTIVITY_ID'!)
    source.updates['last_activities_cache_json'] = json_dumps(
      {k: v for k, v in cache.items() if k.split()[-1] in silo_activity_ids})

    self.backfeed(source, responses, activities=activities)

    source.updates.update({'last_polled': source.last_poll_attempt,
                           'poll_status': 'ok'})
    if etag and etag != source.last_activities_etag:
      source.updates['last_activities_etag'] = etag

    #
    # Possibly refetch updated syndication urls.
    #
    # if the author has added syndication urls since the first time
    # original_post_discovery ran, we'll miss them. this cleanup task will
    # periodically check for updated urls. only kicks in if the author has
    # *ever* published a rel=syndication url
    if source.should_refetch():
      logging.info('refetching h-feed for source %s', source.label())
      relationships = original_post_discovery.refetch(source)

      now = util.now_fn()
      source.updates['last_hfeed_refetch'] = now

      if relationships:
        logging.info('refetch h-feed found new rel=syndication relationships: %s',
                     relationships)
        try:
          self.repropagate_old_responses(source, relationships)
        except BaseException as e:
          if ('BadRequestError' in str(e.__class__) or
              'Timeout' in str(e.__class__) or
              util.is_connection_failure(e)):
            logging.info('Timeout while repropagating responses.', stack_info=True)
          else:
            raise
    else:
      logging.info(
          'skipping refetch h-feed. last-syndication-url %s, last-refetch %s',
          source.last_syndication_url, source.last_hfeed_refetch)

  def backfeed(self, source, responses=None, activities=None):
    """Processes responses and activities and generates propagate tasks.

    Stores property names and values to update in source.updates.

    Args:
      source: Source
      responses: dict mapping AS response id to AS object
      activities: dict mapping AS activity id to AS object
    """
    if responses is None:
      responses = {}
    if activities is None:
      activities = {}

    # Cache to make sure we only fetch the author's h-feed(s) the
    # first time we see it
    fetched_hfeeds = set()

    # narrow down to just public activities
    public = {}
    private = {}
    for id, activity in activities.items():
      (public if source.is_activity_public(activity) else private)[id] = activity
    logging.info('Found %d public activities: %s', len(public), public.keys())
    logging.info('Found %d private activities: %s', len(private), private.keys())

    last_public_post = (source.last_public_post or util.EPOCH).isoformat()
    public_published = util.trim_nulls(
      [a.get('object', {}).get('published') for a in public.values()])
    if public_published:
      max_published = max(public_published)
      if max_published > last_public_post:
        last_public_post = max_published
        source.updates['last_public_post'] = \
          util.as_utc(util.parse_iso8601(max_published))

    source.updates['recent_private_posts'] = \
      len([a for a in private.values()
           if a.get('object', {}).get('published', util.EPOCH_ISO) > last_public_post])

    #
    # Step 2: extract responses, store their activities in response['activities']
    #
    # WARNING: this creates circular references in link posts found by search
    # queries in step 1, since they are their own activity. We use
    # prune_activity() and prune_response() in step 4 to remove these before
    # serializing to JSON.
    #
    for id, activity in public.items():
      obj = activity.get('object') or activity

      # handle user mentions
      user_id = source.user_tag_id()
      if obj.get('author', {}).get('id') != user_id and activity.get('verb') != 'share':
        for tag in obj.get('tags', []):
          urls = tag.get('urls')
          if tag.get('objectType') == 'person' and tag.get('id') == user_id and urls:
            activity['originals'], activity['mentions'] = \
              original_post_discovery.discover(
                source, activity, fetch_hfeed=True,
                include_redirect_sources=False,
                already_fetched_hfeeds=fetched_hfeeds)
            activity['mentions'].update(u.get('value') for u in urls)
            responses[id] = activity
            break

      # handle quote mentions
      for att in obj.get('attachments', []):
        if (att.get('objectType') in ('note', 'article')
                and att.get('author', {}).get('id') == source.user_tag_id()):
          # now that we've confirmed that one exists, OPD will dig
          # into the actual attachments
          if 'originals' not in activity or 'mentions' not in activity:
            activity['originals'], activity['mentions'] = \
              original_post_discovery.discover(
                source, activity, fetch_hfeed=True,
                include_redirect_sources=False,
                already_fetched_hfeeds=fetched_hfeeds)
          responses[id] = activity
          break

      # extract replies, likes, reactions, reposts, and rsvps
      replies = obj.get('replies', {}).get('items', [])
      tags = obj.get('tags', [])
      likes = [t for t in tags if Response.get_type(t) == 'like']
      reactions = [t for t in tags if Response.get_type(t) == 'react']
      reposts = [t for t in tags if Response.get_type(t) == 'repost']
      rsvps = Source.get_rsvps_from_event(obj)

      # coalesce responses. drop any without ids
      for resp in replies + likes + reactions + reposts + rsvps:
        id = resp.get('id')
        if not id:
          logging.error('Skipping response without id: %s', json_dumps(resp, indent=2))
          continue

        if source.is_blocked(resp):
          logging.info('Skipping response by blocked user: %s',
                       json_dumps(resp.get('author') or resp.get('actor'), indent=2))
          continue

        resp.setdefault('activities', []).append(activity)

        # when we find two responses with the same id, the earlier one may have
        # come from a link post or user mention, and this one is probably better
        # since it probably came from the user's activity, so prefer this one.
        # background: https://github.com/snarfed/bridgy/issues/533
        existing = responses.get(id)
        if existing:
          if source.gr_source.activity_changed(resp, existing, log=True):
            logging.warning('Got two different versions of same response!\n%s\n%s',
                            existing, resp)
          resp['activities'].extend(existing.get('activities', []))

        responses[id] = resp

    #
    # Step 3: filter out responses we've already seen
    #
    # seen responses (JSON objects) for each source are stored in its entity.
    unchanged_responses = []
    if source.seen_responses_cache_json:
      for seen in json_loads(source.seen_responses_cache_json):
        id = seen['id']
        resp = responses.get(id)
        if resp and not source.gr_source.activity_changed(seen, resp, log=True):
          unchanged_responses.append(seen)
          del responses[id]

    #
    # Step 4: store new responses and enqueue propagate tasks
    #
    pruned_responses = []
    source.blocked_ids = None

    for id, resp in responses.items():
      resp_type = Response.get_type(resp)
      activities = resp.pop('activities', [])
      if not activities and resp_type == 'post':
        activities = [resp]
      too_long = set()
      urls_to_activity = {}
      for i, activity in enumerate(activities):
        # we'll usually have multiple responses for the same activity, and the
        # objects in resp['activities'] are shared, so cache each activity's
        # discovered webmention targets inside its object.
        if 'originals' not in activity or 'mentions' not in activity:
          activity['originals'], activity['mentions'] = \
            original_post_discovery.discover(
              source, activity, fetch_hfeed=True,
              include_redirect_sources=False,
              already_fetched_hfeeds=fetched_hfeeds)

        targets = original_post_discovery.targets_for_response(
          resp, originals=activity['originals'], mentions=activity['mentions'])
        if targets:
          logging.info('%s has %d webmention target(s): %s', activity.get('url'),
                       len(targets), ' '.join(targets))
          # new response to propagate! load block list if we haven't already
          if source.blocked_ids is None:
            source.load_blocklist()

        for t in targets:
          if len(t) <= _MAX_STRING_LENGTH:
            urls_to_activity[t] = i
          else:
            logging.info('Giving up on target URL over %s chars! %s',
                         _MAX_STRING_LENGTH, t)
            too_long.add(t[:_MAX_STRING_LENGTH - 4] + '...')

      # store/update response entity. the prune_*() calls are important to
      # remove circular references in link responses, which are their own
      # activities. details in the step 2 comment above.
      pruned_response = util.prune_response(resp)
      pruned_responses.append(pruned_response)
      resp_entity = Response(
        id=id,
        source=source.key,
        activities_json=[json_dumps(util.prune_activity(a, source))
                         for a in activities],
        response_json=json_dumps(pruned_response),
        type=resp_type,
        unsent=list(urls_to_activity.keys()),
        failed=list(too_long),
        original_posts=resp.get('originals', []))
      if urls_to_activity and len(activities) > 1:
        resp_entity.urls_to_activity=json_dumps(urls_to_activity)
      resp_entity.get_or_save(source, restart=self.RESTART_EXISTING_TASKS)

    # update cache
    if pruned_responses:
      source.updates['seen_responses_cache_json'] = json_dumps(
        pruned_responses + unchanged_responses)

  def repropagate_old_responses(self, source, relationships):
    """Find old Responses that match a new SyndicatedPost and repropagate them.

    We look through as many responses as we can until the datastore query expires.

    Args:
      source: :class:`models.Source`
      relationships: refetch result
    """
    for response in (Response.query(Response.source == source.key)
                     .order(-Response.updated)):
      new_orig_urls = set()
      for activity_json in response.activities_json:
        activity = json_loads(activity_json)
        activity_url = activity.get('url') or activity.get('object', {}).get('url')
        if not activity_url:
          logging.warning('activity has no url %s', activity_json)
          continue

        activity_url = source.canonicalize_url(activity_url, activity=activity)
        if not activity_url:
          continue

        # look for activity url in the newly discovered list of relationships
        for relationship in relationships.get(activity_url, []):
          # won't re-propagate if the discovered link is already among
          # these well-known upstream duplicates
          if (relationship.original in response.sent or
              relationship.original in response.original_posts):
            logging.info(
              '%s found a new rel=syndication link %s -> %s, but the '
              'relationship had already been discovered by another method',
              response.label(), relationship.original, relationship.syndication)
          else:
            logging.info(
              '%s found a new rel=syndication link %s -> %s, and '
              'will be repropagated with a new target!',
              response.label(), relationship.original, relationship.syndication)
            new_orig_urls.add(relationship.original)

      if new_orig_urls:
        # re-open a previously 'complete' propagate task
        response.status = 'new'
        response.unsent.extend(list(new_orig_urls))
        response.put()
        response.add_task()


class Discover(Poll):
  """Task handler that fetches and processes new responses to a single post.

  Request parameters:

  * source_key: string key of source entity
  * post_id: string, silo post id(s)

  Inserts a propagate task for each response that hasn't been seen before.

  Original feature request: https://github.com/snarfed/bridgy/issues/579
  """
  RESTART_EXISTING_TASKS = True
  TRANSIENT_ERROR_HTTP_CODES = ('400', '404')

  handle_exception = util.background_handle_exception

  def post(self):
    logging.debug('Params: %s', list(self.request.params.items()))

    type = self.request.get('type')
    if type:
      assert type in ('event',)

    source = self.source = util.load_source(self)
    if not source or source.status == 'disabled' or 'listen' not in source.features:
      logging.error('Source not found or disabled. Dropping task.')
      return
    logging.info('Source: %s %s, %s', source.label(), source.key_id(),
                 source.bridgy_url(self))

    post_id = util.get_required_param(self, 'post_id')
    source.updates = {}

    if type == 'event':
      activities = [source.gr_source.get_event(post_id)]
    else:
      activities = source.get_activities(
        fetch_replies=True, fetch_likes=True, fetch_shares=True,
        activity_id=post_id, user_id=source.key_id())

    if not activities or not activities[0]:
      logging.info('Post %s not found.', post_id)
      return
    assert len(activities) == 1, activities
    activity = activities[0]
    activities = {activity['id']: activity}
    self.backfeed(source, responses=activities, activities=activities)

    obj = activity.get('object') or activity
    in_reply_to = util.get_first(obj, 'inReplyTo')
    if in_reply_to:
      parsed = util.parse_tag_uri(in_reply_to.get('id', ''))  # TODO: fall back to url
      if parsed:
        util.add_discover_task(source, parsed[1])


class SendWebmentions(webapp2.RequestHandler):
  """Abstract base task handler that can send webmentions.

  Attributes:

  * entity: :class:`models.Webmentions` subclass instance (set in :meth:`lease_entity`)
  * source: :class:`models.Source` entity (set in :meth:`send_webmentions`)
  """
  # request deadline (10m) plus some padding
  LEASE_LENGTH = datetime.timedelta(minutes=12)
  TRANSIENT_ERROR_HTTP_CODES = ()

  handle_exception = util.background_handle_exception

  def source_url(self, target_url):
    """Return the source URL to use for a given target URL.

    Subclasses must implement.

    Args:
      target_url: string

    Returns:
      string
    """
    raise NotImplementedError()

  def send_webmentions(self):
    """Tries to send each unsent webmention in self.entity.

    Uses :meth:`source_url()` to determine the source parameter for each
    webmention.

    :meth:`lease()` *must* be called before this!
    """
    logging.info('Starting %s', self.entity.label())

    try:
      self.do_send_webmentions()
    except:
      logging.info('Propagate task failed', stack_info=True)
      self.release('error')
      raise

  def do_send_webmentions(self):
    urls = self.entity.unsent + self.entity.error + self.entity.failed
    unsent = set()
    self.entity.error = []
    self.entity.failed = []

    for orig_url in urls:
      # recheck the url here since the checks may have failed during the poll
      # or streaming add.
      url, domain, ok = util.get_webmention_target(orig_url)
      if ok:
        if len(url) <= _MAX_STRING_LENGTH:
          unsent.add(url)
        else:
          logging.info('Giving up on target URL over %s chars! %s',
                       _MAX_STRING_LENGTH, url)
          self.entity.failed.append(orig_url)
    self.entity.unsent = sorted(unsent)

    while self.entity.unsent:
      target = self.entity.unsent.pop(0)
      source_url = self.source_url(target)
      logging.info('Webmention from %s to %s', source_url, target)

      # see if we've cached webmention discovery for this domain. the cache
      # value is a string URL endpoint if discovery succeeded, NO_ENDPOINT if
      # no endpoint was ofund.
      cache_key = util.webmention_endpoint_cache_key(target)
      endpoint = util.webmention_endpoint_cache.get(cache_key)
      if endpoint:
        logging.info('Using cached webmention endpoint %r: %s', cache_key, endpoint)

      # send! and handle response or error
      try:
        resp = None
        headers = util.request_headers(source=self.source)
        if not endpoint:
          endpoint, resp = webmention.discover(target, headers=headers)
          with util.webmention_endpoint_cache_lock:
            util.webmention_endpoint_cache[cache_key] = endpoint or NO_ENDPOINT

        if endpoint and endpoint != NO_ENDPOINT:
          logging.info('Sending...')
          resp = webmention.send(endpoint, source_url, target, timeout=999,
                                 headers=headers)
          logging.info('Sent! %s', resp)
          self.record_source_webmention(endpoint, target)
          self.entity.sent.append(target)
        else:
          logging.info('Giving up this target.')
          self.entity.skipped.append(target)

      except ValueError:
        logging.info('Bad URL; giving up this target.')
        self.entity.skipped.append(target)

      except BaseException as e:
        logging.info('', exc_info=True)
        # Give up on 4XX and DNS errors; we don't expect retries to succeed.
        code, _ = util.interpret_http_exception(e)
        if (code and code.startswith('4')) or 'DNS lookup failed' in str(e):
          logging.info('Giving up this target.')
          self.entity.failed.append(target)
        else:
          self.fail(f'Error sending to endpoint: {resp}', level=logging.INFO)
          self.entity.error.append(target)

      if target in self.entity.unsent:
        self.entity.unsent.remove(target)

    if self.entity.error:
      logging.info('Propagate task failed')
      self.release('error')
    else:
      self.complete()

  @ndb.transactional()
  def lease(self, key):
    """Attempts to acquire and lease the :class:`models.Webmentions` entity.

    Also loads and sets `self.source`, and returns False if the source doesn't
    exist or is disabled.

    TODO: unify with :meth:`complete()`

    Args:
      key: :class:`ndb.Key`

    Returns: True on success, False or None otherwise
    """
    self.entity = key.get()

    if self.entity is None:
      return self.fail('no entity!')
    elif self.entity.status == 'complete':
      # let this task return 200 and finish
      logging.warning('duplicate task already propagated this')
      return
    elif (self.entity.status == 'processing' and
          util.now_fn() < self.entity.leased_until):
      return self.fail('duplicate task is currently processing!')

    self.source = self.entity.source.get()
    if not self.source or self.source.status == 'disabled':
      logging.error('Source not found or disabled. Dropping task.')
      return False
    logging.info('Source: %s %s, %s', self.source.label(), self.source.key_id(),
                 self.source.bridgy_url(self))

    assert self.entity.status in ('new', 'processing', 'error'), self.entity.status
    self.entity.status = 'processing'
    self.entity.leased_until = util.now_fn() + self.LEASE_LENGTH
    self.entity.put()
    return True

  @ndb.transactional()
  def complete(self):
    """Attempts to mark the :class:`models.Webmentions` entity completed.

    Returns True on success, False otherwise.
    """
    existing = self.entity.key.get()
    if existing is None:
      self.fail('entity disappeared!', level=logging.ERROR)
    elif existing.status == 'complete':
      # let this task return 200 and finish
      logging.warning('another task stole and finished this. did my lease expire?')
    elif self.entity.status == 'complete':
      # let this task return 200 and finish
      logging.error('i already completed this task myself somehow?! '
                    'https://github.com/snarfed/bridgy/issues/610')
    elif existing.status == 'new':
      self.fail('went backward from processing to new!', level=logging.ERROR)
    else:
      assert existing.status == 'processing', existing.status
      assert self.entity.status == 'processing', self.entity.status
      self.entity.status = 'complete'
      self.entity.put()
      return True

    return False

  @ndb.transactional()
  def release(self, new_status):
    """Attempts to unlease the :class:`models.Webmentions` entity.

    Args:
      new_status: string
    """
    existing = self.entity.key.get()
    if existing and existing.status == 'processing':
      self.entity.status = new_status
      self.entity.leased_until = None
      self.entity.put()

  def fail(self, message, level=logging.WARNING):
    """Fills in an error response status code and message.
    """
    self.error(util.ERROR_HTTP_RETURN_CODE)
    logging.log(level, message)
    self.response.out.write(message)

  @ndb.transactional()
  def record_source_webmention(self, endpoint, target):
    """Sets this source's last_webmention_sent and maybe webmention_endpoint.

    Args:
      endpoint: str, URL
      target: str, URL
    """
    self.source = self.source.key.get()
    logging.info('Setting last_webmention_sent')
    self.source.last_webmention_sent = util.now_fn()

    if (endpoint != self.source.webmention_endpoint and
        util.domain_from_link(target) in self.source.domains):
      logging.info('Also setting webmention_endpoint to %s (discovered in %s; was %s)',
                   endpoint, target, self.source.webmention_endpoint)
      self.source.webmention_endpoint = endpoint

    self.source.put()


class PropagateResponse(SendWebmentions):
  """Task handler that sends webmentions for a :class:`models.Response`.

  Attributes:

  * activities: parsed :attr:`models.Response.activities_json` list

  Request parameters:

  * response_key: string key of :class:`models.Response` entity
  """

  def post(self):
    self.request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
    logging.debug('Params: %s', list(self.request.params.items()))
    if not self.lease(ndb.Key(urlsafe=self.request.params['response_key'])):
      return

    source = self.source
    poll_estimate = self.entity.created - datetime.timedelta(seconds=61)
    logging.info('Created by this poll: %s/%s', util.host_url(self),
                 logs.url(poll_estimate, source.key))

    self.activities = [json_loads(a) for a in self.entity.activities_json]
    response_obj = json_loads(self.entity.response_json)
    if (not source.is_activity_public(response_obj) or
        not all(source.is_activity_public(a) for a in self.activities)):
      logging.info('Response or activity is non-public. Dropping.')
      self.complete()
      return

    self.send_webmentions()

  def source_url(self, target_url):
    # determine which activity to use
    try:
      activity = self.activities[0]
      if self.entity.urls_to_activity:
        urls_to_activity = json_loads(self.entity.urls_to_activity)
        if urls_to_activity:
          activity = self.activities[urls_to_activity[target_url]]
    except (KeyError, IndexError):
      logging.warning("""\
Hit https://github.com/snarfed/bridgy/issues/237 KeyError!
target url %s not in urls_to_activity: %s
activities: %s""", target_url, self.entity.urls_to_activity, self.activities)
      self.abort(util.ERROR_HTTP_RETURN_CODE)

    # generate source URL
    id = activity['id']
    parsed = util.parse_tag_uri(id)
    post_id = parsed[1] if parsed else id
    parts = [util.host_url(self), self.entity.type, self.source.SHORT_NAME,
            self.source.key.string_id(), post_id]

    if self.entity.type != 'post':
      # parse and add response id. (we know Response key ids are always tag URIs)
      _, response_id = util.parse_tag_uri(self.entity.key.string_id())
      reaction_id = response_id
      if self.entity.type in ('like', 'react', 'repost', 'rsvp'):
        response_id = response_id.split('_')[-1]  # extract responder user id
      parts.append(response_id)
      if self.entity.type == 'react':
        parts.append(reaction_id)

    return '/'.join(parts)


class PropagateBlogPost(SendWebmentions):
  """Task handler that sends webmentions for a :class:`models.BlogPost`.

  Request parameters:

  * key: string key of :class:`models.BlogPost` entity
  """

  def post(self):
    logging.debug('Params: %s', list(self.request.params.items()))

    if not self.lease(ndb.Key(urlsafe=self.request.params['key'])):
      return

    to_send = set()
    for url in self.entity.unsent:
      url, domain, ok = util.get_webmention_target(url)
      # skip "self" links to this blog's domain
      if ok and domain not in self.source.domains:
        to_send.add(url)

    self.entity.unsent = list(to_send)
    self.send_webmentions()

  def source_url(self, target_url):
    return self.entity.key.id()


application = webutil_handlers.ndb_context_middleware(
  webapp2.WSGIApplication([
    ('/_ah/queue/poll(-now)?', Poll),
    ('/_ah/queue/discover', Discover),
    ('/_ah/queue/propagate', PropagateResponse),
    ('/_ah/queue/propagate-blogpost', PropagateBlogPost),
    ('/_ah/(start|stop|warmup)', util.NoopHandler),
  ] + cron.ROUTES, debug=appengine_info.DEBUG), client=ndb_client)