def test_poll(self):
"""A normal poll task."""
self.assertEqual(0, models.Response.query().count())
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
self.post_task()
self.assertEqual(9, models.Response.query().count())
self.assert_responses()
source = self.sources[0].key.get()
self.assertEqual(NOW, source.last_polled)
tasks = self.taskqueue_stub.GetTasks('propagate')
for task in tasks:
self.assertEqual('/_ah/queue/propagate', task['url'])
keys = set(ndb.Key(urlsafe=testutil.get_task_params(t)['response_key'])
for t in tasks)
self.assert_equals(keys, set(r.key for r in self.responses))
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
self.assertEqual('/_ah/queue/poll', tasks[0]['url'])
self.assert_task_eta(FakeSource.FAST_POLL)
params = testutil.get_task_params(tasks[0])
self.assert_equals(source.key.urlsafe(), params['source_key'])
def assert_propagate_task(self, queue='propagate'):
tasks = self.taskqueue_stub.GetTasks('propagate-blogpost')
self.assertEqual(1, len(tasks))
key = testutil.get_task_params(tasks[0])['key']
self.assertEqual(self.blogposts[0].key, ndb.Key(urlsafe=key))
self.assertEqual('/_ah/queue/propagate-blogpost', tasks[0]['url'])
self.taskqueue_stub.FlushQueue('propagate-blogpost')
def test_replace_poll_tasks(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
now = datetime.datetime.now()
# a bunch of sources, one needs a new poll task
five_min_ago = now - datetime.timedelta(minutes=5)
day_and_half_ago = now - datetime.timedelta(hours=36)
month_ago = now - datetime.timedelta(days=30)
defaults = {
'features': ['listen'],
'last_webmention_sent': day_and_half_ago,
}
sources = [
# doesn't need a new poll task
FakeSource.new(None, last_poll_attempt=now, **defaults).put(),
FakeSource.new(None, last_poll_attempt=five_min_ago, **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
# need a new poll task
FakeSource.new(None, status='enabled', **defaults).put(),
# not signed up for listen
FakeSource.new(None, last_webmention_sent=day_and_half_ago).put(),
# never sent a webmention, past grace period. last polled is older than 2x
# fast poll, but within 2x slow poll.
FakeSource.new(None, features=['listen'], created=month_ago,
last_poll_attempt=day_and_half_ago).put(),
]
resp = cron.application.get_response('/cron/replace_poll_tasks')
self.assertEqual(200, resp.status_int)
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
self.assert_equals(sources[4].urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def test_discover_url_site_post_syndication_links(self):
self.expect_requests_get('http://si.te/123', """
<div class="h-entry">
foo
<a class="u-syndication" href="http://fa.ke/222"></a>
<a class="u-syndication" href="http://other/silo"></a>
<a class="u-syndication" href="http://fa.ke/post/444"></a>
</div>""")
self.mox.ReplayAll()
self.assertEqual(0, SyndicatedPost.query().count())
self.check_discover('http://si.te/123',
'Discovering now. Refresh in a minute to see the results!')
self.assertItemsEqual([
{'https://fa.ke/222': 'http://si.te/123'},
{'https://fa.ke/post/444': 'http://si.te/123'},
], [{sp.syndication: sp.original} for sp in models.SyndicatedPost.query()])
tasks = self.taskqueue_stub.GetTasks('discover')
key = self.source.key.urlsafe()
self.assertEqual([
{'source_key': key, 'post_id': '222'},
{'source_key': key, 'post_id': '444'},
], [testutil.get_task_params(task) for task in tasks])
now = util.now_fn()
source = self.source.key.get()
self.assertEqual(now, source.last_syndication_url)
def test_create_new_already_exists(self):
long_ago = datetime.datetime(year=1901, month=2, day=3)
props = {
'created': long_ago,
'last_webmention_sent': long_ago + datetime.timedelta(days=1),
'last_polled': long_ago + datetime.timedelta(days=2),
'last_hfeed_fetch': long_ago + datetime.timedelta(days=3),
'last_syndication_url': long_ago + datetime.timedelta(days=4),
'superfeedr_secret': 'asdfqwert',
}
FakeSource.new(None, features=['listen'], **props).put()
self.assert_equals(['listen'], FakeSource.query().get().features)
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': 'http://foo.com/'}))
auth_entity.put()
self._test_create_new(auth_entity=auth_entity, features=['publish'])
source = FakeSource.query().get()
self.assert_equals(['listen', 'publish'], source.features)
for prop, value in props.items():
self.assert_equals(value, getattr(source, prop), prop)
self.assert_equals(
{"Updated fake (FakeSource). Try previewing a post from your web site!"},
self.handler.messages)
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_replace_poll_tasks(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
now = datetime.datetime.now()
# a bunch of sources, one needs a new poll task
five_min_ago = now - datetime.timedelta(minutes=5)
day_and_half_ago = now - datetime.timedelta(hours=36)
month_ago = now - datetime.timedelta(days=30)
defaults = {
'features': ['listen'],
'last_webmention_sent': day_and_half_ago,
}
sources = [
# doesn't need a new poll task
FakeSource.new(None, last_poll_attempt=now, **defaults).put(),
FakeSource.new(None, last_poll_attempt=five_min_ago, **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
# need a new poll task
FakeSource.new(None, status='enabled', **defaults).put(),
# not signed up for listen
FakeSource.new(None, last_webmention_sent=day_and_half_ago).put(),
# never sent a webmention, past grace period. last polled is older than 2x
# fast poll, but within 2x slow poll.
FakeSource.new(None, features=['listen'], created=month_ago,
last_poll_attempt=day_and_half_ago).put(),
]
resp = cron.application.get_response('/cron/replace_poll_tasks')
self.assertEqual(200, resp.status_int)
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
self.assert_equals(sources[4].urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def test_subscribe(self):
expected = {
"hub.mode": "subscribe",
"hub.topic": "fake feed url",
"hub.callback": "http://localhost/fake/notify/foo.com",
"format": "json",
"retrieve": "true",
}
item_a = {"permalinkUrl": "A", "content": "a http://a.com a"}
item_b = {"permalinkUrl": "B", "summary": "b http://b.com b"}
feed = json.dumps({"items": [item_a, {}, item_b]})
self.expect_requests_post(superfeedr.PUSH_API_URL, feed, data=expected, auth=mox.IgnoreArg())
self.mox.ReplayAll()
superfeedr.subscribe(self.source, self.handler)
posts = list(BlogPost.query())
self.assert_entities_equal(
[
BlogPost(id="A", source=self.source.key, feed_item=item_a, unsent=["http://a.com"]),
BlogPost(id="B", source=self.source.key, feed_item=item_b, unsent=["http://b.com"]),
],
posts,
ignore=("created", "updated"),
)
tasks = self.taskqueue_stub.GetTasks("propagate-blogpost")
self.assert_equals(
[{"key": posts[0].key.urlsafe()}, {"key": posts[1].key.urlsafe()}],
[testutil.get_task_params(t) for t in tasks],
)
def assert_propagate_task(self, queue='propagate'):
tasks = self.taskqueue_stub.GetTasks('propagate-blogpost')
self.assertEqual(1, len(tasks))
key = testutil.get_task_params(tasks[0])['key']
self.assertEqual(self.blogposts[0].key, ndb.Key(urlsafe=key))
self.assertEqual('/_ah/queue/propagate-blogpost', tasks[0]['url'])
self.taskqueue_stub.FlushQueue('propagate-blogpost')
def test_subscribe(self):
expected = {
'hub.mode': 'subscribe',
'hub.topic': 'fake feed url',
'hub.callback': 'http://localhost/fake/notify/foo.com',
'format': 'json',
'retrieve': 'true',
}
item_a = {'permalinkUrl': 'A', 'content': 'a http://a.com a'}
item_b = {'permalinkUrl': 'B', 'summary': 'b http://b.com b'}
feed = json.dumps({'items': [item_a, {}, item_b]})
self.expect_requests_post(superfeedr.PUSH_API_URL, feed,
data=expected, auth=mox.IgnoreArg())
self.mox.ReplayAll()
superfeedr.subscribe(self.source, self.handler)
posts = list(BlogPost.query())
self.assert_entities_equal(
[BlogPost(id='A', source=self.source.key, feed_item=item_a,
unsent=['http://a.com']),
BlogPost(id='B', source=self.source.key, feed_item=item_b,
unsent=['http://b.com']),
], posts,
ignore=('created', 'updated'))
tasks = self.taskqueue_stub.GetTasks('propagate-blogpost')
self.assert_equals([{'key': posts[0].key.urlsafe()},
{'key': posts[1].key.urlsafe()}],
[testutil.get_task_params(t) for t in tasks])
def assert_propagate_task(self):
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEqual(1, len(tasks))
self.assertEqual(self.responses[0].key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assertEqual('/_ah/queue/propagate', tasks[0]['url'])
self.taskqueue_stub.FlushQueue('propagate')
def test_create_new_already_exists(self):
long_ago = datetime.datetime(year=1901, month=2, day=3)
props = {
'created': long_ago,
'last_webmention_sent': long_ago + datetime.timedelta(days=1),
'last_polled': long_ago + datetime.timedelta(days=2),
'last_hfeed_refetch': long_ago + datetime.timedelta(days=3),
'last_syndication_url': long_ago + datetime.timedelta(days=4),
'superfeedr_secret': 'asdfqwert',
}
FakeSource.new(None, features=['listen'], **props).put()
self.assert_equals(['listen'], FakeSource.query().get().features)
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(id='x',
user_json=json.dumps(
{'url': 'http://foo.com/'}))
auth_entity.put()
self._test_create_new(auth_entity=auth_entity, features=['publish'])
source = FakeSource.query().get()
self.assert_equals(['listen', 'publish'], source.features)
for prop, value in props.items():
self.assert_equals(value, getattr(source, prop), prop)
self.assert_equals(
{
"Updated fake (FakeSource). Try previewing a post from your web site!"
}, self.handler.messages)
task_params = testutil.get_task_params(
self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_discover_url_site_post_last_feed_syndication_url(self):
now = util.now_fn()
self.source.last_feed_syndication_url = now
self.source.put()
self.expect_requests_get(
'http://si.te/123', """
<div class="h-entry">
<a class="u-syndication" href="http://fa.ke/222"></a>
</div>""")
self.mox.ReplayAll()
self.check_discover(
'http://si.te/123',
'Discovering now. Refresh in a minute to see the results!')
tasks = self.taskqueue_stub.GetTasks('discover')
key = self.source.key.urlsafe()
self.assertEqual([{
'source_key': key,
'post_id': '222'
}], [testutil.get_task_params(task) for task in tasks])
source = self.source.key.get()
self.assertEqual(now, source.last_syndication_url)
def assert_propagate_task(self):
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEqual(1, len(tasks))
self.assertEqual(self.responses[0].key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assertEqual('/_ah/queue/propagate', tasks[0]['url'])
self.taskqueue_stub.FlushQueue('propagate')
def assert_blogposts(self, expected):
got = list(BlogPost.query())
self.assert_entities_equal(expected, got, ignore=('created', 'updated'))
tasks = self.taskqueue_stub.GetTasks('propagate-blogpost')
self.assert_equals([{'key': post.key.urlsafe()} for post in expected],
[testutil.get_task_params(t) for t in tasks])
def test_create_new(self):
self.assertEqual(0, FakeSource.query().count())
self._test_create_new(features=['listen'])
msg = "Added fake (FakeSource). Refresh to see what we've found!"
self.assert_equals({msg}, self.handler.messages)
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_create_new(self):
self.assertEqual(0, FakeSource.query().count())
self._test_create_new(features=['listen'])
msg = "Added fake (FakeSource). Refresh in a minute to see what we've found!"
self.assert_equals({msg}, self.handler.messages)
for queue in 'poll', 'poll-now':
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks(queue)[0])
self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_poll_now(self):
self.assertEqual([], self.taskqueue_stub.GetTasks("poll"))
key = self.sources[0].key.urlsafe()
resp = app.application.get_response("/poll-now", method="POST", body="key=" + key)
self.assertEquals(302, resp.status_int)
self.assertEquals(self.sources[0].bridgy_url(self.handler), resp.headers["Location"].split("#")[0])
params = testutil.get_task_params(self.taskqueue_stub.GetTasks("poll-now")[0])
self.assertEqual(key, params["source_key"])
def test_do_refetch_hfeed(self):
"""Emulate a situation where we've done posse-post-discovery earlier and
found no rel=syndication relationships for a particular silo URL. Every
two hours or so, we should refetch the author's page and check to see if
any new syndication links have been added or updated.
"""
self.sources[0].domain_urls = ['http://author']
FakeAsSource.DOMAIN = 'source'
self.sources[0].last_syndication_url = NOW - datetime.timedelta(minutes=10)
self.sources[0].last_hfeed_fetch = NOW - datetime.timedelta(hours=2,
minutes=10)
self.sources[0].put()
# pretend we've already done posse-post-discovery for the source
# and checked this permalink and found no back-links
models.SyndicatedPost(parent=self.sources[0].key, original=None,
syndication='https://source/post/url').put()
models.SyndicatedPost(parent=self.sources[0].key,
original='http://author/permalink',
syndication=None).put()
# and all the status have already been sent
for r in self.responses:
r.status = 'complete'
r.put()
self.expect_requests_get('http://author', """
<html class="h-feed">
<a class="h-entry" href="/permalink"></a>
</html>""")
self.expect_requests_get('http://author/permalink', """
<html class="h-entry">
<a class="u-url" href="http://author/permalink"></a>
<a class="u-syndication" href="http://source/post/url"></a>
</html>""")
self.mox.ReplayAll()
self.post_task()
# should have a new SyndicatedPost
relationship = models.SyndicatedPost.query_by_original(
self.sources[0], 'http://author/permalink')
self.assertIsNotNone(relationship)
self.assertEquals('https://source/post/url', relationship.syndication)
# should repropagate all 9 responses
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEquals(9, len(tasks))
# and they should be in reverse creation order
response_keys = [resp.key.urlsafe() for resp in self.responses]
response_keys.reverse()
task_keys = [testutil.get_task_params(task)['response_key']
for task in tasks]
self.assertEquals(response_keys, task_keys)
def test_discover_url_silo_post(self):
self.check_discover('http://fa.ke/123',
'Discovering now. Refresh in a minute to see the results!')
tasks = self.taskqueue_stub.GetTasks('discover')
self.assertEqual(1, len(tasks))
self.assertEqual({
'source_key': self.source.key.urlsafe(),
'post_id': '123',
}, testutil.get_task_params(tasks[0]))
def test_poll_now(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
key = self.sources[0].key.urlsafe()
resp = app.application.get_response('/poll-now', method='POST', body='key=' + key)
self.assertEquals(302, resp.status_int)
self.assertEquals(self.sources[0].bridgy_url(self.handler),
resp.headers['Location'].split('#')[0])
params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual(key, params['source_key'])
def test_retry(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('propagate'))
source = self.sources[0]
source.domain_urls = ['http://orig']
source.last_hfeed_fetch = last_hfeed_fetch = \
testutil.NOW - datetime.timedelta(minutes=1)
source.put()
resp = self.responses[0]
resp.status = 'complete'
resp.unsent = ['http://unsent']
resp.sent = ['http://sent']
resp.error = ['http://error']
resp.failed = ['http://failed']
resp.skipped = ['https://skipped']
# SyndicatedPost with new target URLs
resp.activities_json = [
json.dumps({'object': {'url': 'https://silo/1'}}),
json.dumps({'url': 'https://silo/2', 'object': {'unused': 'ok'}}),
json.dumps({'url': 'https://silo/3'}),
]
resp.put()
models.SyndicatedPost.insert(source, 'https://silo/1', 'https://orig/1')
models.SyndicatedPost.insert(source, 'https://silo/2', 'http://orig/2')
models.SyndicatedPost.insert(source, 'https://silo/3', 'http://orig/3')
# cached webmention endpoint
memcache.set('W https skipped', 'asdf')
key = resp.key.urlsafe()
response = app.application.get_response(
'/retry', method='POST', body='key=' + key)
self.assertEquals(302, response.status_int)
self.assertEquals(source.bridgy_url(self.handler),
response.headers['Location'].split('#')[0])
params = testutil.get_task_params(self.taskqueue_stub.GetTasks('propagate')[0])
self.assertEqual(key, params['response_key'])
# status and URLs should be refreshed
got = resp.key.get()
self.assertEqual('new', got.status)
self.assertItemsEqual(
['http://unsent', 'http://sent', 'https://skipped', 'http://error',
'http://failed', 'https://orig/1', 'http://orig/2', 'http://orig/3'],
got.unsent)
for field in got.sent, got.skipped, got.error, got.failed:
self.assertEqual([], field)
# webmention endpoints for URL domains should be refreshed
self.assertIsNone(memcache.get('W https skipped'))
# shouldn't have refetched h-feed
self.assertEqual(last_hfeed_fetch, source.key.get().last_hfeed_fetch)
def test_non_public_posts(self):
"""Only posts with to: @public should be propagated."""
del self.activities[0]['object']['to']
self.activities[1]['object']['to'] = [{'objectType':'group', 'alias':'@private'}]
self.activities[2]['object']['to'] = [{'objectType':'group', 'alias':'@public'}]
self.post_task()
ids = set()
for task in self.taskqueue_stub.GetTasks('propagate'):
resp_key = ndb.Key(urlsafe=testutil.get_task_params(task)['response_key'])
ids.update(json.loads(a)['id'] for a in resp_key.get().activities_json)
self.assert_equals(ids, set([self.activities[0]['id'], self.activities[2]['id']]))
def test_discover_url_silo_post(self):
self.check_discover(
'http://fa.ke/123',
'Discovering now. Refresh in a minute to see the results!')
tasks = self.taskqueue_stub.GetTasks('discover')
self.assertEqual(1, len(tasks))
self.assertEqual(
{
'source_key': self.source.key.urlsafe(),
'post_id': '123',
}, testutil.get_task_params(tasks[0]))
def test_poll_now(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
key = self.sources[0].key.urlsafe()
resp = app.application.get_response('/poll-now',
method='POST',
body='key=' + key)
self.assertEquals(302, resp.status_int)
self.assertEquals(self.sources[0].bridgy_url(self.handler),
resp.headers['Location'].split('#')[0])
params = testutil.get_task_params(
self.taskqueue_stub.GetTasks('poll-now')[0])
self.assertEqual(key, params['source_key'])
def _test_create_new(self, **kwargs):
FakeSource.create_new(self.handler, domains=['foo'],
domain_urls=['http://foo.com'],
webmention_endpoint='http://x/y',
**kwargs)
self.assertEqual(1, FakeSource.query().count())
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
source = FakeSource.query().get()
self.assertEqual('/_ah/queue/poll', tasks[0]['url'])
self.assertEqual(source.key.urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def _test_create_new(self, **kwargs):
FakeSource.create_new(self.handler, domains=['foo'],
domain_urls=['http://foo.com'],
webmention_endpoint='http://x/y',
**kwargs)
self.assertEqual(1, FakeSource.query().count())
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
source = FakeSource.query().get()
self.assertEqual('/_ah/queue/poll', tasks[0]['url'])
self.assertEqual(source.key.urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
self.assertEqual('fake (FakeSource)', source.label())
def test_crawl_now(self):
source = self.sources[0]
source.domain_urls = ["http://orig"]
source.last_hfeed_refetch = testutil.NOW
source.put()
key = source.key.urlsafe()
response = app.application.get_response("/crawl-now", method="POST", body="key=%s" % key)
self.assertEquals(source.bridgy_url(self.handler), response.headers["Location"].split("#")[0])
self.assertEquals(302, response.status_int)
params = testutil.get_task_params(self.taskqueue_stub.GetTasks("poll-now")[0])
self.assertEqual(key, params["source_key"])
self.assertEqual(models.REFETCH_HFEED_TRIGGER, source.key.get().last_hfeed_refetch)
def test_good(self):
self.expect_requests_get(
'http://foo.com/', """
<html class="h-feed">
<div class="h-entry">
<a class="u-url" href="http://foo.com/post"></a>
<a class="u-syndication" href="https://www.facebook.com/snarfed.org/posts/123"></a>
</div>
</html>""")
self.mox.ReplayAll()
self.handler.receive(self.mail)
self.assert_equals(200, self.response.status_code)
emails = list(FacebookEmail.query())
self.assertEquals(1, len(emails))
self.assert_equals('SMTP-123-xyz', emails[0].key.id())
self.assert_equals(self.fea.key, emails[0].source)
self.assert_equals([COMMENT_EMAIL_USERNAME], emails[0].htmls)
resp_id = EMAIL_COMMENT_OBJ_USERNAME['id']
self.assert_equals(ndb.Key('Response', resp_id), emails[0].response)
expected = Response(
id=resp_id,
source=self.fea.key,
type='comment',
response_json=json.dumps(EMAIL_COMMENT_OBJ_USERNAME),
activities_json=[
json.dumps({
'id': '123',
'numeric_id': '123',
'url': 'https://www.facebook.com/212038/posts/123',
'author': {
'id': 'snarfed.org'
},
})
],
unsent=['http://foo.com/post'])
self.assert_entities_equal([expected],
list(Response.query()),
ignore=('created', 'updated'))
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEquals(1, len(tasks))
self.assert_equals(expected.key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assert_equals(EMAIL_COMMENT_OBJ_USERNAME,
self.fea.get_comment('123_789'))
def test_handle_feed(self):
item_a = {"permalinkUrl": "A", "content": "a http://a.com http://foo.com/self/link b"}
superfeedr.handle_feed(json.dumps({"items": [item_a]}), self.source)
posts = list(BlogPost.query())
self.assert_entities_equal(
[
BlogPost(id="A", source=self.source.key, feed_item=item_a, unsent=["http://a.com"])
], # self link should be discarded
posts,
ignore=("created", "updated"),
)
tasks = self.taskqueue_stub.GetTasks("propagate-blogpost")
self.assertEqual(1, len(tasks))
self.assert_equals(posts[0].key.urlsafe(), testutil.get_task_params(tasks[0])["key"])
def test_crawl_now(self):
source = self.sources[0]
source.domain_urls = ['http://orig']
source.last_hfeed_fetch = testutil.NOW
source.put()
key = source.key.urlsafe()
response = app.application.get_response(
'/crawl-now', method='POST', body='key=%s' % key)
self.assertEquals(source.bridgy_url(self.handler),
response.headers['Location'].split('#')[0])
self.assertEquals(302, response.status_int)
params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll-now')[0])
self.assertEqual(key, params['source_key'])
self.assertEqual(models.REFETCH_HFEED_TRIGGER, source.key.get().last_hfeed_fetch)
def test_handle_feed(self):
item_a = {'permalinkUrl': 'A',
'content': 'a http://a.com http://foo.com/self/link b'}
superfeedr.handle_feed(json.dumps({'items': [item_a]}), self.source)
posts = list(BlogPost.query())
self.assert_entities_equal(
[BlogPost(id='A', source=self.source.key, feed_item=item_a,
unsent=['http://a.com'])], # self link should be discarded
posts,
ignore=('created', 'updated'))
tasks = self.taskqueue_stub.GetTasks('propagate-blogpost')
self.assertEqual(1, len(tasks))
self.assert_equals(posts[0].key.urlsafe(),
testutil.get_task_params(tasks[0])['key'])
def test_discover_url_site_post_syndication_links(self):
self.expect_requests_get(
'http://si.te/123', """
<div class="h-entry">
foo
<a class="u-syndication" href="http://fa.ke/222"></a>
<a class="u-syndication" href="http://other/silo"></a>
<a class="u-syndication" href="http://fa.ke/post/444"></a>
</div>""")
self.mox.ReplayAll()
self.assertEqual(0, SyndicatedPost.query().count())
self.check_discover(
'http://si.te/123',
'Discovering now. Refresh in a minute to see the results!')
self.assertItemsEqual([
{
'https://fa.ke/222': 'http://si.te/123'
},
{
'https://fa.ke/post/444': 'http://si.te/123'
},
], [{
sp.syndication: sp.original
} for sp in models.SyndicatedPost.query()])
tasks = self.taskqueue_stub.GetTasks('discover')
key = self.source.key.urlsafe()
self.assertEqual([
{
'source_key': key,
'post_id': '222'
},
{
'source_key': key,
'post_id': '444'
},
], [testutil.get_task_params(task) for task in tasks])
now = util.now_fn()
source = self.source.key.get()
self.assertEqual(now, source.last_syndication_url)
def test_discover_url_site_post_last_feed_syndication_url(self):
now = util.now_fn()
self.source.last_feed_syndication_url = now
self.source.put()
self.expect_requests_get('http://si.te/123', """
<div class="h-entry">
<a class="u-syndication" href="http://fa.ke/222"></a>
</div>""")
self.mox.ReplayAll()
self.check_discover('http://si.te/123',
'Discovering now. Refresh in a minute to see the results!')
tasks = self.taskqueue_stub.GetTasks('discover')
key = self.source.key.urlsafe()
self.assertEqual([{'source_key': key, 'post_id': '222'}],
[testutil.get_task_params(task) for task in tasks])
source = self.source.key.get()
self.assertEqual(now, source.last_syndication_url)
def test_crawl_now(self):
source = self.sources[0]
source.domain_urls = ['http://orig']
source.last_hfeed_refetch = source.last_feed_syndication_url = testutil.NOW
source.put()
key = source.key.urlsafe()
response = app.application.get_response('/crawl-now',
method='POST',
body='key=%s' % key)
self.assertEquals(source.bridgy_url(self.handler),
response.headers['Location'].split('#')[0])
self.assertEquals(302, response.status_int)
params = testutil.get_task_params(
self.taskqueue_stub.GetTasks('poll-now')[0])
self.assertEqual(key, params['source_key'])
source = source.key.get()
self.assertEqual(models.REFETCH_HFEED_TRIGGER,
source.last_hfeed_refetch)
self.assertIsNone(source.last_feed_syndication_url)
def test_get_or_save(self):
self.sources[0].put()
response = self.responses[0]
self.assertEqual(0, Response.query().count())
self.assertEqual(0, len(self.taskqueue_stub.GetTasks('propagate')))
# new. should add a propagate task.
saved = response.get_or_save()
self.assertEqual(response.key, saved.key)
self.assertEqual(response.source, saved.source)
self.assertEqual('comment', saved.type)
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEqual(1, len(tasks))
self.assertEqual(response.key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assertEqual('/_ah/queue/propagate', tasks[0]['url'])
# existing. no new task.
same = saved.get_or_save()
self.assertEqual(saved.source, same.source)
self.assertEqual(1, len(tasks))
def test_retry(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('propagate'))
source = self.sources[0]
source.domain_urls = ['http://orig']
source.last_hfeed_refetch = last_hfeed_refetch = \
testutil.NOW - datetime.timedelta(minutes=1)
source.put()
resp = self.responses[0]
resp.status = 'complete'
resp.unsent = ['http://unsent']
resp.sent = ['http://sent']
resp.error = ['http://error']
resp.failed = ['http://failed']
resp.skipped = ['https://skipped']
# SyndicatedPost with new target URLs
resp.activities_json = [
json.dumps({'object': {
'url': 'https://fa.ke/1'
}}),
json.dumps({
'url': 'https://fa.ke/2',
'object': {
'unused': 'ok'
}
}),
json.dumps({'url': 'https://fa.ke/3'}),
]
resp.put()
SyndicatedPost.insert(source, 'https://fa.ke/1', 'https://orig/1')
SyndicatedPost.insert(source, 'https://fa.ke/2', 'http://orig/2')
SyndicatedPost.insert(source, 'https://fa.ke/3', 'http://orig/3')
# cached webmention endpoint
memcache.set('W https skipped', 'asdf')
key = resp.key.urlsafe()
response = app.application.get_response('/retry',
method='POST',
body='key=' + key)
self.assertEquals(302, response.status_int)
self.assertEquals(source.bridgy_url(self.handler),
response.headers['Location'].split('#')[0])
params = testutil.get_task_params(
self.taskqueue_stub.GetTasks('propagate')[0])
self.assertEqual(key, params['response_key'])
# status and URLs should be refreshed
got = resp.key.get()
self.assertEqual('new', got.status)
self.assertItemsEqual([
'http://unsent/', 'http://sent/', 'https://skipped/',
'http://error/', 'http://failed/', 'https://orig/1',
'http://orig/2', 'http://orig/3'
], got.unsent)
for field in got.sent, got.skipped, got.error, got.failed:
self.assertEqual([], field)
# webmention endpoints for URL domains should be refreshed
self.assertIsNone(memcache.get('W https skipped'))
# shouldn't have refetched h-feed
self.assertEqual(last_hfeed_refetch,
source.key.get().last_hfeed_refetch)
def test_retry(self):
self.assertEqual([], self.taskqueue_stub.GetTasks("propagate"))
source = self.sources[0]
source.domain_urls = ["http://orig"]
source.last_hfeed_refetch = last_hfeed_refetch = testutil.NOW - datetime.timedelta(minutes=1)
source.put()
resp = self.responses[0]
resp.status = "complete"
resp.unsent = ["http://unsent"]
resp.sent = ["http://sent"]
resp.error = ["http://error"]
resp.failed = ["http://failed"]
resp.skipped = ["https://skipped"]
# SyndicatedPost with new target URLs
resp.activities_json = [
json.dumps({"object": {"url": "https://silo/1"}}),
json.dumps({"url": "https://silo/2", "object": {"unused": "ok"}}),
json.dumps({"url": "https://silo/3"}),
]
resp.put()
models.SyndicatedPost.insert(source, "https://silo/1", "https://orig/1")
models.SyndicatedPost.insert(source, "https://silo/2", "http://orig/2")
models.SyndicatedPost.insert(source, "https://silo/3", "http://orig/3")
# cached webmention endpoint
memcache.set("W https skipped", "asdf")
key = resp.key.urlsafe()
response = app.application.get_response("/retry", method="POST", body="key=" + key)
self.assertEquals(302, response.status_int)
self.assertEquals(source.bridgy_url(self.handler), response.headers["Location"].split("#")[0])
params = testutil.get_task_params(self.taskqueue_stub.GetTasks("propagate")[0])
self.assertEqual(key, params["response_key"])
# status and URLs should be refreshed
got = resp.key.get()
self.assertEqual("new", got.status)
self.assertItemsEqual(
[
"http://unsent",
"http://sent",
"https://skipped",
"http://error",
"http://failed",
"https://orig/1",
"http://orig/2",
"http://orig/3",
],
got.unsent,
)
for field in got.sent, got.skipped, got.error, got.failed:
self.assertEqual([], field)
# webmention endpoints for URL domains should be refreshed
self.assertIsNone(memcache.get("W https skipped"))
# shouldn't have refetched h-feed
self.assertEqual(last_hfeed_refetch, source.key.get().last_hfeed_refetch)
