def test_rate_limiting_errors(self):
"""Finish the task on rate limiting errors."""
try:
for err in (InstagramAPIError('503', 'Rate limited', '...'),
apiclient.errors.HttpError(httplib2.Response({'status': 429}), ''),
urllib2.HTTPError('url', 403, 'msg', {}, None)):
self.mox.UnsetStubs()
self.mox.StubOutWithMock(FakeSource, 'get_activities_response')
FakeSource.get_activities_response(
count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True,
fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(),
).AndRaise(err)
self.mox.ReplayAll()
self.post_task()
source = self.sources[0].key.get()
self.assertEqual('error', source.status)
self.mox.VerifyAll()
# should have inserted a new poll task
polls = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(polls))
self.assertEqual('/_ah/queue/poll', polls[0]['url'])
polls = self.taskqueue_stub.FlushQueue('poll')
finally:
self.mox.UnsetStubs()
def test_create_new_domain(self):
"""If the source has a URL set, extract its domain."""
for user_json in None, {}, {'url': 'not<a>url'}, {'url': 'http://t.co/foo'}:
auth_entity = None
if user_json is not None:
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(user_json))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEqual([], source.domains)
self.assertEqual([], source.domain_urls)
# good URLs
for url in ('http://foo.com/bar', 'https://www.foo.com/bar',
'http://foo.com/\nhttp://baz.com/',
'http://FoO.cOm', # should be normalized to lowercase
):
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': url}))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals([url.split('\n')[0]], source.domain_urls)
self.assertEquals(['foo.com'], source.domains)
# also look in urls field
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'url': 'not<a>url',
'urls': [{'value': 'also<not>'}, {'value': 'http://foo.com/'}],
}))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://foo.com/'], source.domain_urls)
self.assertEquals(['foo.com'], source.domains)
def test_create_new_already_exists(self):
long_ago = datetime.datetime(year=1901, month=2, day=3)
props = {
'created': long_ago,
'last_webmention_sent': long_ago + datetime.timedelta(days=1),
'last_polled': long_ago + datetime.timedelta(days=2),
'last_hfeed_fetch': long_ago + datetime.timedelta(days=3),
'last_syndication_url': long_ago + datetime.timedelta(days=4),
'superfeedr_secret': 'asdfqwert',
}
FakeSource.new(None, features=['listen'], **props).put()
self.assert_equals(['listen'], FakeSource.query().get().features)
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': 'http://foo.com/'}))
auth_entity.put()
self._test_create_new(auth_entity=auth_entity, features=['publish'])
source = FakeSource.query().get()
self.assert_equals(['listen', 'publish'], source.features)
for prop, value in props.items():
self.assert_equals(value, getattr(source, prop), prop)
self.assert_equals(
{"Updated fake (FakeSource). Try previewing a post from your web site!"},
self.handler.messages)
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_create_new_already_exists(self):
long_ago = datetime.datetime(year=1901, month=2, day=3)
props = {
'created': long_ago,
'last_webmention_sent': long_ago + datetime.timedelta(days=1),
'last_polled': long_ago + datetime.timedelta(days=2),
'last_hfeed_refetch': long_ago + datetime.timedelta(days=3),
'last_syndication_url': long_ago + datetime.timedelta(days=4),
'superfeedr_secret': 'asdfqwert',
}
FakeSource.new(None, features=['listen'], **props).put()
self.assert_equals(['listen'], FakeSource.query().get().features)
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(id='x',
user_json=json.dumps(
{'url': 'http://foo.com/'}))
auth_entity.put()
self._test_create_new(auth_entity=auth_entity, features=['publish'])
source = FakeSource.query().get()
self.assert_equals(['listen', 'publish'], source.features)
for prop, value in props.items():
self.assert_equals(value, getattr(source, prop), prop)
self.assert_equals(
{
"Updated fake (FakeSource). Try previewing a post from your web site!"
}, self.handler.messages)
task_params = testutil.get_task_params(
self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_create_new_rereads_domains(self):
FakeSource.new(None,
features=['listen'],
domain_urls=['http://foo'],
domains=['foo']).put()
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(id='x',
user_json=json.dumps({
'urls': [{
'value':
'http://bar'
}, {
'value':
'http://baz'
}]
}))
self.expect_webmention_requests_get('http://bar/',
'no webmention endpoint',
verify=False)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://bar/', 'http://baz/'], source.domain_urls)
self.assertEquals(['bar', 'baz'], source.domains)
def test_in_blocklist(self):
self.mox.StubOutWithMock(FakeSource, 'is_blocked')
FakeSource.is_blocked(mox.IgnoreArg()).AndReturn(True)
self.mox.ReplayAll()
orig = self.check_response('/comment/fake/%s/000/111',
expected_status=410)
def test_create_new_webmention(self):
"""We should subscribe to webmention sources in Superfeedr."""
self.expect_requests_get('http://primary/', 'no webmention endpoint',
verify=False)
self.mox.StubOutWithMock(superfeedr, 'subscribe')
superfeedr.subscribe(mox.IsA(FakeSource), self.handler)
self.mox.ReplayAll()
FakeSource.create_new(self.handler, features=['webmention'],
domains=['primary/'], domain_urls=['http://primary/'])
def test_create_new_domain(self):
"""If the source has a URL set, extract its domain."""
# bad URLs
for user_json in (None, {}, {'url': 'not<a>url'},
# t.co is in the webmention blacklist
{'url': 'http://t.co/foo'}):
auth_entity = None
if user_json is not None:
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(user_json))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEqual([], source.domains)
self.assertEqual([], source.domain_urls)
self.expect_requests_get('http://foo.com')
self.expect_requests_get('https://www.foo.com')
self.expect_requests_get('https://baj')
self.mox.ReplayAll()
# good URLs
for url in ('http://foo.com/bar', 'https://www.foo.com/bar',
'http://FoO.cOm/', # should be normalized to lowercase
):
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': url}))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals([url.lower()], source.domain_urls)
self.assertEquals(['foo.com'], source.domains)
# multiple good URLs and one that's in the webmention blacklist
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps({
'url': 'http://foo.org',
'urls': [{'value': u} for u in
'http://bar.com', 'http://t.co/x', 'http://baz',
# utm_* query params should be stripped
'https://baj/biff?utm_campaign=x&utm_source=y'],
}))
auth_entity.put()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://foo.org/', 'http://bar.com/', 'http://baz/',
'https://baj/biff'],
source.domain_urls)
self.assertEquals(['foo.org', 'bar.com', 'baz', 'baj'], source.domains)
# a URL that redirects
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': 'http://orig'}))
auth_entity.put()
self.expect_requests_head('http://orig', redirected_url='http://final')
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://final/'], source.domain_urls)
self.assertEquals(['final'], source.domains)
def test_poll_error(self):
"""If anything goes wrong, the source status should be set to 'error'."""
self.mox.StubOutWithMock(FakeSource, 'get_activities_response')
FakeSource.get_activities_response(
count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True,
fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(),
).AndRaise(Exception('foo'))
self.mox.ReplayAll()
self.assertRaises(Exception, self.post_task)
source = self.sources[0].key.get()
self.assertEqual('error', source.status)
def _test_create_new(self, **kwargs):
FakeSource.create_new(self.handler, domains=['foo'],
domain_urls=['http://foo.com'],
webmention_endpoint='http://x/y',
**kwargs)
self.assertEqual(1, FakeSource.query().count())
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
source = FakeSource.query().get()
self.assertEqual('/_ah/queue/poll', tasks[0]['url'])
self.assertEqual(source.key.urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def test_create_new_webmention(self):
"""We should subscribe to webmention sources in Superfeedr."""
self.expect_webmention_requests_get('http://primary/',
'no webmention endpoint',
verify=False)
self.mox.StubOutWithMock(superfeedr, 'subscribe')
superfeedr.subscribe(mox.IsA(FakeSource), self.handler)
self.mox.ReplayAll()
FakeSource.create_new(self.handler,
features=['webmention'],
domains=['primary/'],
domain_urls=['http://primary/'])
def _test_create_new(self, **kwargs):
FakeSource.create_new(self.handler, domains=['foo'],
domain_urls=['http://foo.com'],
webmention_endpoint='http://x/y',
**kwargs)
self.assertEqual(1, FakeSource.query().count())
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
source = FakeSource.query().get()
self.assertEqual('/_ah/queue/poll', tasks[0]['url'])
self.assertEqual(source.key.urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
self.assertEqual('fake (FakeSource)', source.label())
def test_create_new_rereads_domains(self):
FakeSource.new(None, features=['listen'],
domain_urls=['http://foo'], domains=['foo']).put()
FakeSource.string_id_counter -= 1
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': 'http://bar'}, {'value': 'http://baz'}]}))
self.expect_requests_get('http://bar', 'no webmention endpoint',
verify=False)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://bar', 'http://baz'], source.domain_urls)
self.assertEquals(['bar', 'baz'], source.domains)
def test_create_new_webmention(self):
"""We should subscribe to webmention sources in Superfeedr."""
self.expect_webmention_requests_get('http://primary/', 'no webmention endpoint',
verify=False)
self.mox.StubOutWithMock(superfeedr, 'subscribe')
def check_source(source):
assert isinstance(source, FakeSource)
assert source.is_saved
return True
superfeedr.subscribe(mox.Func(check_source), self.handler)
self.mox.ReplayAll()
FakeSource.create_new(self.handler, features=['webmention'],
domains=['primary/'], domain_urls=['http://primary/'])
def test_create_new_webmention(self):
"""We should subscribe to webmention sources in Superfeedr."""
self.expect_webmention_requests_get('http://primary/', 'no webmention endpoint',
verify=False)
self.mox.StubOutWithMock(superfeedr, 'subscribe')
def check_source(source):
assert isinstance(source, FakeSource)
assert source.is_saved
return True
superfeedr.subscribe(mox.Func(check_source), self.handler)
self.mox.ReplayAll()
FakeSource.create_new(self.handler, features=['webmention'],
domains=['primary/'], domain_urls=['http://primary/'])
def test_get_comment_injects_web_site_urls_into_user_mentions(self):
source = FakeSource.new(None,
domain_urls=['http://site1/', 'http://site2/'])
source.put()
user_id = 'tag:fa.ke,2013:%s' % source.key.id()
FakeGrSource.comment = {
'id': 'tag:fa.ke,2013:a1-b2.c3',
'tags': [
{
'id': 'tag:fa.ke,2013:nobody'
},
{
'id': user_id
},
],
}
# check that we inject their web sites
self.assert_equals(
{
'id': 'tag:fa.ke,2013:%s' % source.key.id(),
'urls': [{
'value': 'http://site1/'
}, {
'value': 'http://site2/'
}],
},
super(FakeSource, source).get_comment('x')['tags'][1])
def test_get_activities_injects_web_site_urls_into_user_mentions(self):
source = FakeSource.new(None,
domain_urls=['http://site1/', 'http://site2/'])
source.put()
mention = {
'object': {
'tags': [{
'objectType': 'person',
'id': 'tag:fa.ke,2013:%s' % source.key.id(),
'url': 'https://fa.ke/me',
}, {
'objectType': 'person',
'id': 'tag:fa.ke,2013:bob',
}],
},
}
FakeGrSource.activities = [mention]
# check that we inject their web sites
got = super(FakeSource, source).get_activities_response()
mention['object']['tags'][0]['urls'] = [{
'value': 'http://site1/'
}, {
'value': 'http://site2/'
}]
self.assert_equals([mention], got['items'])
def test_disable_source_on_deauthorized(self):
"""If the source raises DisableSource, disable it.
"""
source = self.sources[0]
self.mox.StubOutWithMock(FakeSource, 'get_activities_response')
FakeSource.get_activities_response(
count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True,
fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(),
).AndRaise(models.DisableSource)
self.mox.ReplayAll()
source.status = 'enabled'
source.put()
self.post_task()
source = source.key.get()
self.assertEqual('disabled', source.status)
def setUp(self):
super(SyndicatedPostTest, self).setUp()
self.source = FakeSource.new(None)
self.source.put()
self.relationships = []
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/post/url',
syndication='http://silo/post/url'))
# two syndication for the same original
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/post/url',
syndication='http://silo/another/url'))
# two originals for the same syndication
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/another/post',
syndication='http://silo/post/url'))
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original=None,
syndication='http://silo/no-original'))
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/no-syndication',
syndication=None))
for r in self.relationships:
r.put()
def setUp(self):
super(SyndicatedPostTest, self).setUp()
self.source = FakeSource.new(None)
self.source.put()
self.relationships = []
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/post/url',
syndication='http://silo/post/url'))
# two syndication for the same original
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/post/url',
syndication='http://silo/another/url'))
# two originals for the same syndication
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/another/post',
syndication='http://silo/post/url'))
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original=None,
syndication='http://silo/no-original'))
self.relationships.append(
SyndicatedPost(parent=self.source.key,
original='http://original/no-syndication',
syndication=None))
for r in self.relationships:
r.put()
def test_registration_with_user_url(self):
"""Run through an authorization back and forth with a custom user url
provided to the auth mechanism
"""
encoded_state = urllib.quote_plus(
'{"callback":"http://withknown.com/bridgy_callback","feature":"listen",'
'"operation":"add","user_url":"https://kylewm.com"}')
application = webapp2.WSGIApplication([
('/fakesource/start', testutil.FakeStartHandler),
('/fakesource/add', testutil.FakeAddHandler),
])
self.expect_webmention_requests_get(
'https://kylewm.com/',
response='<html><link rel="webmention" href="/webmention"></html>',
verify=False)
self.mox.ReplayAll()
resp = application.get_response(
'/fakesource/start',
method='POST',
body=urllib.urlencode({
'feature': 'listen',
'callback': 'http://withknown.com/bridgy_callback',
'user_url': 'https://kylewm.com',
}))
expected_auth_url = 'http://fake/auth/url?' + urllib.urlencode({
'redirect_uri':
'http://localhost/fakesource/add?state=' + encoded_state,
})
self.assert_equals(302, resp.status_code)
self.assert_equals(expected_auth_url, resp.headers['location'])
resp = application.get_response(
'/fakesource/add?state=' + encoded_state +
'&oauth_token=fake-token&oauth_token_secret=fake-secret')
self.assert_equals(302, resp.status_code)
self.assert_equals(
'http://withknown.com/bridgy_callback?' + urllib.urlencode([
('result', 'success'),
('key', ndb.Key('FakeSource', '0123456789').urlsafe()),
('user', 'http://localhost/fake/0123456789')
]), resp.headers['location'])
self.assertEquals(
'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/',
resp.headers['Set-Cookie'])
source = FakeSource.get_by_id('0123456789')
self.assertTrue(source)
self.assert_equals('Fake User', source.name)
self.assert_equals(['listen'], source.features)
self.assert_equals(['https://kylewm.com/', 'http://fakeuser.com/'],
source.domain_urls)
self.assert_equals(['kylewm.com', 'fakeuser.com'], source.domains)
def test_create_new(self):
self.assertEqual(0, FakeSource.query().count())
self._test_create_new(features=['listen'])
msg = "Added fake (FakeSource). Refresh to see what we've found!"
self.assert_equals({msg}, self.handler.messages)
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0])
self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_replace_poll_tasks(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
now = datetime.datetime.now()
# a bunch of sources, one needs a new poll task
five_min_ago = now - datetime.timedelta(minutes=5)
day_and_half_ago = now - datetime.timedelta(hours=36)
month_ago = now - datetime.timedelta(days=30)
defaults = {
'features': ['listen'],
'last_webmention_sent': day_and_half_ago,
}
sources = [
# doesn't need a new poll task
FakeSource.new(None, last_poll_attempt=now, **defaults).put(),
FakeSource.new(None, last_poll_attempt=five_min_ago, **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
# need a new poll task
FakeSource.new(None, status='enabled', **defaults).put(),
# not signed up for listen
FakeSource.new(None, last_webmention_sent=day_and_half_ago).put(),
# never sent a webmention, past grace period. last polled is older than 2x
# fast poll, but within 2x slow poll.
FakeSource.new(None, features=['listen'], created=month_ago,
last_poll_attempt=day_and_half_ago).put(),
]
resp = cron.application.get_response('/cron/replace_poll_tasks')
self.assertEqual(200, resp.status_int)
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
self.assert_equals(sources[4].urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def test_replace_poll_tasks(self):
self.assertEqual([], self.taskqueue_stub.GetTasks('poll'))
now = datetime.datetime.now()
# a bunch of sources, one needs a new poll task
five_min_ago = now - datetime.timedelta(minutes=5)
day_and_half_ago = now - datetime.timedelta(hours=36)
month_ago = now - datetime.timedelta(days=30)
defaults = {
'features': ['listen'],
'last_webmention_sent': day_and_half_ago,
}
sources = [
# doesn't need a new poll task
FakeSource.new(None, last_poll_attempt=now, **defaults).put(),
FakeSource.new(None, last_poll_attempt=five_min_ago, **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
FakeSource.new(None, status='disabled', **defaults).put(),
# need a new poll task
FakeSource.new(None, status='enabled', **defaults).put(),
# not signed up for listen
FakeSource.new(None, last_webmention_sent=day_and_half_ago).put(),
# never sent a webmention, past grace period. last polled is older than 2x
# fast poll, but within 2x slow poll.
FakeSource.new(None, features=['listen'], created=month_ago,
last_poll_attempt=day_and_half_ago).put(),
]
resp = cron.application.get_response('/cron/replace_poll_tasks')
self.assertEqual(200, resp.status_int)
tasks = self.taskqueue_stub.GetTasks('poll')
self.assertEqual(1, len(tasks))
self.assert_equals(sources[4].urlsafe(),
testutil.get_task_params(tasks[0])['source_key'])
def test_registration_with_user_url(self):
"""Run through an authorization back and forth with a custom user url
provided to the auth mechanism
"""
encoded_state = urllib.quote_plus(
'{"callback":"http://withknown.com/bridgy_callback","feature":"listen",'
'"operation":"add","user_url":"https://kylewm.com"}')
application = webapp2.WSGIApplication([
('/fakesource/start', testutil.FakeStartHandler),
('/fakesource/add', testutil.FakeAddHandler),
])
self.expect_webmention_requests_get(
'https://kylewm.com/',
response='<html><link rel="webmention" href="/webmention"></html>',
verify=False)
self.mox.ReplayAll()
resp = application.get_response(
'/fakesource/start', method='POST', body=urllib.urlencode({
'feature': 'listen',
'callback': 'http://withknown.com/bridgy_callback',
'user_url': 'https://kylewm.com',
}))
expected_auth_url = 'http://fake/auth/url?' + urllib.urlencode({
'redirect_uri': 'http://localhost/fakesource/add?state='
+ encoded_state,
})
self.assert_equals(302, resp.status_code)
self.assert_equals(expected_auth_url, resp.headers['location'])
resp = application.get_response(
'/fakesource/add?state=' + encoded_state +
'&oauth_token=fake-token&oauth_token_secret=fake-secret')
self.assert_equals(302, resp.status_code)
self.assert_equals(
'http://withknown.com/bridgy_callback?' + urllib.urlencode([
('result', 'success'),
('key', ndb.Key('FakeSource', '0123456789').urlsafe()),
('user', 'http://localhost/fake/0123456789')]),
resp.headers['location'])
self.assertEquals(
'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/',
resp.headers['Set-Cookie'])
source = FakeSource.get_by_id('0123456789')
self.assertTrue(source)
self.assert_equals('Fake User', source.name)
self.assert_equals(['listen'], source.features)
self.assert_equals(['https://kylewm.com/', 'http://fakeuser.com/'],
source.domain_urls)
self.assert_equals(['kylewm.com', 'fakeuser.com'], source.domains)
def test_is_beta_user(self):
source = FakeSource.new(self.handler)
self.assertFalse(source.is_beta_user())
self.mox.stubs.Set(util, 'BETA_USER_PATHS', set())
self.assertFalse(source.is_beta_user())
self.mox.stubs.Set(util, 'BETA_USER_PATHS', set([source.bridgy_path()]))
self.assertTrue(source.is_beta_user())
def test_has_bridgy_webmention_endpoint(self):
source = FakeSource.new(None)
for endpoint, has in ((None, False),
('http://foo', False ),
('https://brid.gy/webmention/fake', True),
('https://www.brid.gy/webmention/fake', True),
):
source.webmention_endpoint = endpoint
self.assertEquals(has, source.has_bridgy_webmention_endpoint(), endpoint)
def test_verify_without_webmention_endpoint(self):
self.expect_webmention_requests_get(
'http://primary/', 'no webmention endpoint here!', verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://primary/'], domains=['primary'])
source.verify()
self.assertIsNone(source.webmention_endpoint)
def test_last_activity_id(self):
"""We should store the last activity id seen and then send it as min_id."""
self.sources[0].set_activities(list(reversed(self.activities)))
self.post_task()
source = self.sources[0].key.get()
self.assertEqual('c', source.last_activity_id)
source.last_polled = util.EPOCH
source.put()
self.mox.StubOutWithMock(FakeSource, 'get_activities_response')
FakeSource.get_activities_response(
count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True,
fetch_shares=True, etag=None, min_id='c', cache=mox.IgnoreArg(),
).AndReturn({'items': []})
self.mox.ReplayAll()
self.post_task()
def test_create_new(self):
self.assertEqual(0, FakeSource.query().count())
self._test_create_new(features=['listen'])
msg = "Added fake (FakeSource). Refresh in a minute to see what we've found!"
self.assert_equals({msg}, self.handler.messages)
for queue in 'poll', 'poll-now':
task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks(queue)[0])
self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_has_bridgy_webmention_endpoint(self):
source = FakeSource.new(None)
for endpoint, has in ((None, False),
('http://foo', False ),
('https://brid.gy/webmention/fake', True),
('https://www.brid.gy/webmention/fake', True),
):
source.webmention_endpoint = endpoint
self.assertEquals(has, source.has_bridgy_webmention_endpoint(), endpoint)
def test_verify_without_webmention_endpoint(self):
self.expect_requests_get('http://primary/', 'no webmention endpoint here!',
verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://primary/'], domains=['primary'])
source.verify()
self.assertIsNone(source.webmention_endpoint)
def test_create_new_domain_url_path_fails(self):
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': 'http://flaky/foo'}]}))
self.expect_requests_get('http://flaky', status_code=500)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://flaky/foo'], source.domain_urls)
self.assertEquals(['flaky'], source.domains)
def test_get_comment(self):
comment_obj = {'objectType': 'comment', 'content': 'qwert'}
source = FakeSource.new(None)
source.as_source = self.mox.CreateMock(as_source.Source)
source.as_source.get_comment('123', activity_id=None, activity_author_id=None
).AndReturn(comment_obj)
self.mox.ReplayAll()
self.assert_equals(comment_obj, source.get_comment('123'))
def test_create_new_domain_url_path_connection_fails(self):
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': 'http://flaky/foo'}]}))
self.expect_requests_get('http://flaky').AndRaise(
requests.ConnectionError('DNS lookup failed for URL: http://bad/'))
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://flaky/foo'], source.domain_urls)
self.assertEquals(['flaky'], source.domains)
def test_registration_callback(self):
"""Run through an authorization back and forth and make sure that
the external callback makes it all the way through.
"""
encoded_state = urllib.quote_plus(
'{"callback":"http://withknown.com/bridgy_callback",' '"feature":"listen","operation":"add"}'
)
application = webapp2.WSGIApplication(
[("/fakesource/start", testutil.FakeStartHandler), ("/fakesource/add", testutil.FakeAddHandler)]
)
self.expect_webmention_requests_get(
u"http://fakeuser.com/", response='<html><link rel="webmention" href="/webmention"></html>', verify=False
)
self.mox.ReplayAll()
resp = application.get_response(
"/fakesource/start",
method="POST",
body=urllib.urlencode({"feature": "listen", "callback": "http://withknown.com/bridgy_callback"}),
)
expected_auth_url = "http://fake/auth/url?" + urllib.urlencode(
{"redirect_uri": "http://localhost/fakesource/add?state=" + encoded_state}
)
self.assert_equals(302, resp.status_code)
self.assert_equals(expected_auth_url, resp.headers["location"])
resp = application.get_response(
"/fakesource/add?state=" + encoded_state + "&oauth_token=fake-token&oauth_token_secret=fake-secret"
)
self.assert_equals(302, resp.status_code)
self.assert_equals(
"http://withknown.com/bridgy_callback?"
+ urllib.urlencode(
[
("result", "success"),
("key", ndb.Key("FakeSource", "0123456789").urlsafe()),
("user", "http://localhost/fake/0123456789"),
]
),
resp.headers["location"],
)
self.assertEquals(
'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/', resp.headers["Set-Cookie"]
)
source = FakeSource.get_by_id("0123456789")
self.assertTrue(source)
self.assert_equals("Fake User", source.name)
self.assert_equals(["listen"], source.features)
def test_create_new_dedupes_domains(self):
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': 'http://foo'},
{'value': 'https://foo/'},
{'value': 'http://foo/'},
{'value': 'http://foo'},
]}))
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['https://foo/'], source.domain_urls)
self.assertEquals(['foo'], source.domains)
def test_create_new_dedupes_domains(self):
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': 'http://foo'},
{'value': 'https://foo/'},
{'value': 'http://foo/'},
{'value': 'http://foo'},
]}))
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['https://foo/'], source.domain_urls)
self.assertEquals(['foo'], source.domains)
def test_etag(self):
"""If we see an ETag, we should send it with the next get_activities()."""
self.sources[0]._set('etag', '"my etag"')
self.post_task()
source = self.sources[0].key.get()
self.assertEqual('"my etag"', source.last_activities_etag)
source.last_polled = util.EPOCH
source.put()
self.mox.StubOutWithMock(FakeSource, 'get_activities_response')
FakeSource.get_activities_response(
count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True,
fetch_shares=True, etag='"my etag"', min_id='c', cache=mox.IgnoreArg(),
).AndReturn({'items': [], 'etag': '"new etag"'})
self.mox.ReplayAll()
self.post_task()
source = self.sources[0].key.get()
self.assertEqual('"new etag"', source.last_activities_etag)
def test_verify(self):
# this requests.get is called by webmention-tools
self.expect_webmention_requests_get('http://primary/', """
<html><meta>
<link rel="webmention" href="http://web.ment/ion">
</meta></html>""", verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://primary/'], domains=['primary'])
source.verify()
self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_domain_url_path_fails(self):
auth_entity = testutil.FakeAuthEntity(
id='x',
user_json=json.dumps({'urls': [{
'value': 'http://flaky/foo'
}]}))
self.expect_requests_get('http://flaky', status_code=500)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://flaky/foo'], source.domain_urls)
self.assertEquals(['flaky'], source.domains)
def test_verify(self):
# this requests.get is called by webmention-tools
self.expect_requests_get('http://primary/', """
<html><meta>
<link rel="webmention" href="http://web.ment/ion">
</meta></html>""", verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://primary/'], domains=['primary'])
source.verify()
self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_domain_url_no_root_relme(self):
"""If a profile URL contains a path, check the root for a rel=me to the path."""
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': 'http://site/path'}))
auth_entity.put()
self.expect_requests_get('http://site')
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://site/path'], source.domain_urls)
self.assertEquals(['site'], source.domains)
def test_create_new_domain_url_redirects_to_path(self):
"""If a profile URL is a root that redirects to a path, keep the root."""
auth_entity = testutil.FakeAuthEntity(
id='x', user_json=json.dumps({'url': 'http://site'}))
auth_entity.put()
self.expect_requests_head('http://site', redirected_url='https://site/path')
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://site/'], source.domain_urls)
self.assertEquals(['site'], source.domains)
def test_verify_checks_blacklist(self):
self.expect_webmention_requests_get('http://good/', """
<html><meta>
<link rel="webmention" href="http://web.ment/ion">
</meta></html>""", verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://bad.app/', 'http://good/'],
domains=['bad.app', 'good'])
source.verify()
self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_too_many_domains(self):
urls = ['http://%s/' % i for i in range(10)]
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': u} for u in urls]}))
# we should only check the first 5
for url in urls[:models.MAX_AUTHOR_URLS]:
self.expect_requests_head(url)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(urls, source.domain_urls)
self.assertEquals([str(i) for i in range(10)], source.domains)
def test_put_updates(self):
source = FakeSource.new(None)
source.put()
updates = source.updates = {'status': 'disabled'}
try:
# check that source.updates is preserved through pre-put hook since some
# Source subclasses (e.g. FacebookPage) use it.
FakeSource._pre_put_hook = lambda fake: self.assertEquals(updates, fake.updates)
Source.put_updates(source)
self.assertEquals('disabled', source.key.get().status)
finally:
del FakeSource._pre_put_hook
def test_create_new_too_many_domains(self):
urls = ['http://%s/' % i for i in range(10)]
auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(
{'urls': [{'value': u} for u in urls]}))
# we should only check the first 5
for url in urls[:models.MAX_AUTHOR_URLS]:
self.expect_requests_head(url)
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(urls, source.domain_urls)
self.assertEquals([str(i) for i in range(10)], source.domains)
def test_create_new_domain_url_path_connection_fails(self):
auth_entity = testutil.FakeAuthEntity(
id='x',
user_json=json.dumps({'urls': [{
'value': 'http://flaky/foo'
}]}))
self.expect_requests_get('http://flaky').AndRaise(
requests.ConnectionError('DNS lookup failed for URL: http://bad/'))
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEquals(['http://flaky/foo'], source.domain_urls)
self.assertEquals(['flaky'], source.domains)
def test_put_updates(self):
source = FakeSource.new(None)
source.put()
updates = source.updates = {'status': 'disabled'}
try:
# check that source.updates is preserved through pre-put hook since some
# Source subclasses (e.g. FacebookPage) use it.
FakeSource._pre_put_hook = lambda fake: self.assertEquals(updates, fake.updates)
Source.put_updates(source)
self.assertEquals('disabled', source.key.get().status)
finally:
del FakeSource._pre_put_hook
def test_verify_checks_blacklist(self):
self.expect_webmention_requests_get('http://good/',
"""
<html><meta>
<link rel="webmention" href="http://web.ment/ion">
</meta></html>""",
verify=False)
self.mox.ReplayAll()
source = FakeSource.new(
self.handler,
features=['webmention'],
domain_urls=['http://bad.app/', 'http://good/'],
domains=['bad.app', 'good'])
source.verify()
self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_verify_unicode_characters(self):
"""Older versions of BS4 had an issue where it would check short HTML
documents to make sure the user wasn't accidentally passing a URL,
but converting the utf-8 document to ascii caused exceptions in some cases.
"""
# this requests.get is called by webmention-tools
self.expect_webmention_requests_get(
'http://primary/', """\xef\xbb\xbf<html><head>
<link rel="webmention" href="http://web.ment/ion"></head>
</html>""", verify=False)
self.mox.ReplayAll()
source = FakeSource.new(self.handler, features=['webmention'],
domain_urls=['http://primary/'],
domains=['primary'])
source.verify()
self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_poll_period(self):
source = FakeSource.new(None)
source.put()
self.assertEqual(source.FAST_POLL, source.poll_period())
source.created = datetime.datetime(2000, 1, 1)
self.assertEqual(source.SLOW_POLL, source.poll_period())
now = datetime.datetime.now()
source.last_webmention_sent = now - datetime.timedelta(days=8)
self.assertEqual(source.FAST_POLL * 10, source.poll_period())
source.last_webmention_sent = now
self.assertEqual(source.FAST_POLL, source.poll_period())
source.rate_limited = True
self.assertEqual(source.RATE_LIMITED_POLL, source.poll_period())
def test_should_refetch(self):
source = FakeSource.new(None) # haven't found a synd url yet
self.assertFalse(source.should_refetch())
source.last_hfeed_refetch = models.REFETCH_HFEED_TRIGGER # override
self.assertTrue(source.should_refetch())
source.last_syndication_url = source.last_hfeed_refetch = testutil.NOW # too soon
self.assertFalse(source.should_refetch())
source.last_poll_attempt = testutil.NOW # too soon
self.assertFalse(source.should_refetch())
hour = datetime.timedelta(hours=1)
source.last_hfeed_refetch -= (Source.FAST_REFETCH + hour)
self.assertTrue(source.should_refetch())
source.last_syndication_url -= datetime.timedelta(days=15) # slow refetch
self.assertFalse(source.should_refetch())
source.last_hfeed_refetch -= (Source.SLOW_REFETCH + hour)
self.assertTrue(source.should_refetch())
def test_create_new_publish(self):
"""If a source is publish only, we shouldn't insert a poll task."""
FakeSource.create_new(self.handler, features=['publish'])
self.assertEqual(0, len(self.taskqueue_stub.GetTasks('poll')))
self.assertEqual(0, len(self.taskqueue_stub.GetTasks('poll-now')))
def test_create_new_unicode_chars(self):
"""We should handle unusual unicode chars in the source's name ok."""
# the invisible character in the middle is an unusual unicode character
FakeSource.create_new(self.handler, name=u'a ✁ b')
