def test_rate_limiting_errors(self): """Finish the task on rate limiting errors.""" try: for err in (InstagramAPIError('503', 'Rate limited', '...'), apiclient.errors.HttpError(httplib2.Response({'status': 429}), ''), urllib2.HTTPError('url', 403, 'msg', {}, None)): self.mox.UnsetStubs() self.mox.StubOutWithMock(FakeSource, 'get_activities_response') FakeSource.get_activities_response( count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True, fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(), ).AndRaise(err) self.mox.ReplayAll() self.post_task() source = self.sources[0].key.get() self.assertEqual('error', source.status) self.mox.VerifyAll() # should have inserted a new poll task polls = self.taskqueue_stub.GetTasks('poll') self.assertEqual(1, len(polls)) self.assertEqual('/_ah/queue/poll', polls[0]['url']) polls = self.taskqueue_stub.FlushQueue('poll') finally: self.mox.UnsetStubs()
def test_create_new_domain(self): """If the source has a URL set, extract its domain.""" for user_json in None, {}, {'url': 'not<a>url'}, {'url': 'http://t.co/foo'}: auth_entity = None if user_json is not None: auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(user_json)) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEqual([], source.domains) self.assertEqual([], source.domain_urls) # good URLs for url in ('http://foo.com/bar', 'https://www.foo.com/bar', 'http://foo.com/\nhttp://baz.com/', 'http://FoO.cOm', # should be normalized to lowercase ): auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': url})) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals([url.split('\n')[0]], source.domain_urls) self.assertEquals(['foo.com'], source.domains) # also look in urls field auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'url': 'not<a>url', 'urls': [{'value': 'also<not>'}, {'value': 'http://foo.com/'}], })) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://foo.com/'], source.domain_urls) self.assertEquals(['foo.com'], source.domains)
def test_create_new_already_exists(self): long_ago = datetime.datetime(year=1901, month=2, day=3) props = { 'created': long_ago, 'last_webmention_sent': long_ago + datetime.timedelta(days=1), 'last_polled': long_ago + datetime.timedelta(days=2), 'last_hfeed_fetch': long_ago + datetime.timedelta(days=3), 'last_syndication_url': long_ago + datetime.timedelta(days=4), 'superfeedr_secret': 'asdfqwert', } FakeSource.new(None, features=['listen'], **props).put() self.assert_equals(['listen'], FakeSource.query().get().features) FakeSource.string_id_counter -= 1 auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': 'http://foo.com/'})) auth_entity.put() self._test_create_new(auth_entity=auth_entity, features=['publish']) source = FakeSource.query().get() self.assert_equals(['listen', 'publish'], source.features) for prop, value in props.items(): self.assert_equals(value, getattr(source, prop), prop) self.assert_equals( {"Updated fake (FakeSource). Try previewing a post from your web site!"}, self.handler.messages) task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0]) self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_create_new_already_exists(self): long_ago = datetime.datetime(year=1901, month=2, day=3) props = { 'created': long_ago, 'last_webmention_sent': long_ago + datetime.timedelta(days=1), 'last_polled': long_ago + datetime.timedelta(days=2), 'last_hfeed_refetch': long_ago + datetime.timedelta(days=3), 'last_syndication_url': long_ago + datetime.timedelta(days=4), 'superfeedr_secret': 'asdfqwert', } FakeSource.new(None, features=['listen'], **props).put() self.assert_equals(['listen'], FakeSource.query().get().features) FakeSource.string_id_counter -= 1 auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'url': 'http://foo.com/'})) auth_entity.put() self._test_create_new(auth_entity=auth_entity, features=['publish']) source = FakeSource.query().get() self.assert_equals(['listen', 'publish'], source.features) for prop, value in props.items(): self.assert_equals(value, getattr(source, prop), prop) self.assert_equals( { "Updated fake (FakeSource). Try previewing a post from your web site!" }, self.handler.messages) task_params = testutil.get_task_params( self.taskqueue_stub.GetTasks('poll')[0]) self.assertEqual('1901-02-05-00-00-00', task_params['last_polled'])
def test_create_new_rereads_domains(self): FakeSource.new(None, features=['listen'], domain_urls=['http://foo'], domains=['foo']).put() FakeSource.string_id_counter -= 1 auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps({ 'urls': [{ 'value': 'http://bar' }, { 'value': 'http://baz' }] })) self.expect_webmention_requests_get('http://bar/', 'no webmention endpoint', verify=False) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://bar/', 'http://baz/'], source.domain_urls) self.assertEquals(['bar', 'baz'], source.domains)
def test_in_blocklist(self): self.mox.StubOutWithMock(FakeSource, 'is_blocked') FakeSource.is_blocked(mox.IgnoreArg()).AndReturn(True) self.mox.ReplayAll() orig = self.check_response('/comment/fake/%s/000/111', expected_status=410)
def test_create_new_webmention(self): """We should subscribe to webmention sources in Superfeedr.""" self.expect_requests_get('http://primary/', 'no webmention endpoint', verify=False) self.mox.StubOutWithMock(superfeedr, 'subscribe') superfeedr.subscribe(mox.IsA(FakeSource), self.handler) self.mox.ReplayAll() FakeSource.create_new(self.handler, features=['webmention'], domains=['primary/'], domain_urls=['http://primary/'])
def test_create_new_domain(self): """If the source has a URL set, extract its domain.""" # bad URLs for user_json in (None, {}, {'url': 'not<a>url'}, # t.co is in the webmention blacklist {'url': 'http://t.co/foo'}): auth_entity = None if user_json is not None: auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps(user_json)) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEqual([], source.domains) self.assertEqual([], source.domain_urls) self.expect_requests_get('http://foo.com') self.expect_requests_get('https://www.foo.com') self.expect_requests_get('https://baj') self.mox.ReplayAll() # good URLs for url in ('http://foo.com/bar', 'https://www.foo.com/bar', 'http://FoO.cOm/', # should be normalized to lowercase ): auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': url})) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals([url.lower()], source.domain_urls) self.assertEquals(['foo.com'], source.domains) # multiple good URLs and one that's in the webmention blacklist auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps({ 'url': 'http://foo.org', 'urls': [{'value': u} for u in 'http://bar.com', 'http://t.co/x', 'http://baz', # utm_* query params should be stripped 'https://baj/biff?utm_campaign=x&utm_source=y'], })) auth_entity.put() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://foo.org/', 'http://bar.com/', 'http://baz/', 'https://baj/biff'], source.domain_urls) self.assertEquals(['foo.org', 'bar.com', 'baz', 'baj'], source.domains) # a URL that redirects auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': 'http://orig'})) auth_entity.put() self.expect_requests_head('http://orig', redirected_url='http://final') self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://final/'], source.domain_urls) self.assertEquals(['final'], source.domains)
def test_poll_error(self): """If anything goes wrong, the source status should be set to 'error'.""" self.mox.StubOutWithMock(FakeSource, 'get_activities_response') FakeSource.get_activities_response( count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True, fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(), ).AndRaise(Exception('foo')) self.mox.ReplayAll() self.assertRaises(Exception, self.post_task) source = self.sources[0].key.get() self.assertEqual('error', source.status)
def _test_create_new(self, **kwargs): FakeSource.create_new(self.handler, domains=['foo'], domain_urls=['http://foo.com'], webmention_endpoint='http://x/y', **kwargs) self.assertEqual(1, FakeSource.query().count()) tasks = self.taskqueue_stub.GetTasks('poll') self.assertEqual(1, len(tasks)) source = FakeSource.query().get() self.assertEqual('/_ah/queue/poll', tasks[0]['url']) self.assertEqual(source.key.urlsafe(), testutil.get_task_params(tasks[0])['source_key'])
def test_create_new_webmention(self): """We should subscribe to webmention sources in Superfeedr.""" self.expect_webmention_requests_get('http://primary/', 'no webmention endpoint', verify=False) self.mox.StubOutWithMock(superfeedr, 'subscribe') superfeedr.subscribe(mox.IsA(FakeSource), self.handler) self.mox.ReplayAll() FakeSource.create_new(self.handler, features=['webmention'], domains=['primary/'], domain_urls=['http://primary/'])
def _test_create_new(self, **kwargs): FakeSource.create_new(self.handler, domains=['foo'], domain_urls=['http://foo.com'], webmention_endpoint='http://x/y', **kwargs) self.assertEqual(1, FakeSource.query().count()) tasks = self.taskqueue_stub.GetTasks('poll') self.assertEqual(1, len(tasks)) source = FakeSource.query().get() self.assertEqual('/_ah/queue/poll', tasks[0]['url']) self.assertEqual(source.key.urlsafe(), testutil.get_task_params(tasks[0])['source_key']) self.assertEqual('fake (FakeSource)', source.label())
def test_create_new_rereads_domains(self): FakeSource.new(None, features=['listen'], domain_urls=['http://foo'], domains=['foo']).put() FakeSource.string_id_counter -= 1 auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'urls': [{'value': 'http://bar'}, {'value': 'http://baz'}]})) self.expect_requests_get('http://bar', 'no webmention endpoint', verify=False) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://bar', 'http://baz'], source.domain_urls) self.assertEquals(['bar', 'baz'], source.domains)
def test_create_new_webmention(self): """We should subscribe to webmention sources in Superfeedr.""" self.expect_webmention_requests_get('http://primary/', 'no webmention endpoint', verify=False) self.mox.StubOutWithMock(superfeedr, 'subscribe') def check_source(source): assert isinstance(source, FakeSource) assert source.is_saved return True superfeedr.subscribe(mox.Func(check_source), self.handler) self.mox.ReplayAll() FakeSource.create_new(self.handler, features=['webmention'], domains=['primary/'], domain_urls=['http://primary/'])
def test_get_comment_injects_web_site_urls_into_user_mentions(self): source = FakeSource.new(None, domain_urls=['http://site1/', 'http://site2/']) source.put() user_id = 'tag:fa.ke,2013:%s' % source.key.id() FakeGrSource.comment = { 'id': 'tag:fa.ke,2013:a1-b2.c3', 'tags': [ { 'id': 'tag:fa.ke,2013:nobody' }, { 'id': user_id }, ], } # check that we inject their web sites self.assert_equals( { 'id': 'tag:fa.ke,2013:%s' % source.key.id(), 'urls': [{ 'value': 'http://site1/' }, { 'value': 'http://site2/' }], }, super(FakeSource, source).get_comment('x')['tags'][1])
def test_get_activities_injects_web_site_urls_into_user_mentions(self): source = FakeSource.new(None, domain_urls=['http://site1/', 'http://site2/']) source.put() mention = { 'object': { 'tags': [{ 'objectType': 'person', 'id': 'tag:fa.ke,2013:%s' % source.key.id(), 'url': 'https://fa.ke/me', }, { 'objectType': 'person', 'id': 'tag:fa.ke,2013:bob', }], }, } FakeGrSource.activities = [mention] # check that we inject their web sites got = super(FakeSource, source).get_activities_response() mention['object']['tags'][0]['urls'] = [{ 'value': 'http://site1/' }, { 'value': 'http://site2/' }] self.assert_equals([mention], got['items'])
def test_disable_source_on_deauthorized(self): """If the source raises DisableSource, disable it. """ source = self.sources[0] self.mox.StubOutWithMock(FakeSource, 'get_activities_response') FakeSource.get_activities_response( count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True, fetch_shares=True, etag=None, min_id=None, cache=mox.IgnoreArg(), ).AndRaise(models.DisableSource) self.mox.ReplayAll() source.status = 'enabled' source.put() self.post_task() source = source.key.get() self.assertEqual('disabled', source.status)
def setUp(self): super(SyndicatedPostTest, self).setUp() self.source = FakeSource.new(None) self.source.put() self.relationships = [] self.relationships.append( SyndicatedPost(parent=self.source.key, original='http://original/post/url', syndication='http://silo/post/url')) # two syndication for the same original self.relationships.append( SyndicatedPost(parent=self.source.key, original='http://original/post/url', syndication='http://silo/another/url')) # two originals for the same syndication self.relationships.append( SyndicatedPost(parent=self.source.key, original='http://original/another/post', syndication='http://silo/post/url')) self.relationships.append( SyndicatedPost(parent=self.source.key, original=None, syndication='http://silo/no-original')) self.relationships.append( SyndicatedPost(parent=self.source.key, original='http://original/no-syndication', syndication=None)) for r in self.relationships: r.put()
def test_registration_with_user_url(self): """Run through an authorization back and forth with a custom user url provided to the auth mechanism """ encoded_state = urllib.quote_plus( '{"callback":"http://withknown.com/bridgy_callback","feature":"listen",' '"operation":"add","user_url":"https://kylewm.com"}') application = webapp2.WSGIApplication([ ('/fakesource/start', testutil.FakeStartHandler), ('/fakesource/add', testutil.FakeAddHandler), ]) self.expect_webmention_requests_get( 'https://kylewm.com/', response='<html><link rel="webmention" href="/webmention"></html>', verify=False) self.mox.ReplayAll() resp = application.get_response( '/fakesource/start', method='POST', body=urllib.urlencode({ 'feature': 'listen', 'callback': 'http://withknown.com/bridgy_callback', 'user_url': 'https://kylewm.com', })) expected_auth_url = 'http://fake/auth/url?' + urllib.urlencode({ 'redirect_uri': 'http://localhost/fakesource/add?state=' + encoded_state, }) self.assert_equals(302, resp.status_code) self.assert_equals(expected_auth_url, resp.headers['location']) resp = application.get_response( '/fakesource/add?state=' + encoded_state + '&oauth_token=fake-token&oauth_token_secret=fake-secret') self.assert_equals(302, resp.status_code) self.assert_equals( 'http://withknown.com/bridgy_callback?' + urllib.urlencode([ ('result', 'success'), ('key', ndb.Key('FakeSource', '0123456789').urlsafe()), ('user', 'http://localhost/fake/0123456789') ]), resp.headers['location']) self.assertEquals( 'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/', resp.headers['Set-Cookie']) source = FakeSource.get_by_id('0123456789') self.assertTrue(source) self.assert_equals('Fake User', source.name) self.assert_equals(['listen'], source.features) self.assert_equals(['https://kylewm.com/', 'http://fakeuser.com/'], source.domain_urls) self.assert_equals(['kylewm.com', 'fakeuser.com'], source.domains)
def test_create_new(self): self.assertEqual(0, FakeSource.query().count()) self._test_create_new(features=['listen']) msg = "Added fake (FakeSource). Refresh to see what we've found!" self.assert_equals({msg}, self.handler.messages) task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll')[0]) self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_replace_poll_tasks(self): self.assertEqual([], self.taskqueue_stub.GetTasks('poll')) now = datetime.datetime.now() # a bunch of sources, one needs a new poll task five_min_ago = now - datetime.timedelta(minutes=5) day_and_half_ago = now - datetime.timedelta(hours=36) month_ago = now - datetime.timedelta(days=30) defaults = { 'features': ['listen'], 'last_webmention_sent': day_and_half_ago, } sources = [ # doesn't need a new poll task FakeSource.new(None, last_poll_attempt=now, **defaults).put(), FakeSource.new(None, last_poll_attempt=five_min_ago, **defaults).put(), FakeSource.new(None, status='disabled', **defaults).put(), FakeSource.new(None, status='disabled', **defaults).put(), # need a new poll task FakeSource.new(None, status='enabled', **defaults).put(), # not signed up for listen FakeSource.new(None, last_webmention_sent=day_and_half_ago).put(), # never sent a webmention, past grace period. last polled is older than 2x # fast poll, but within 2x slow poll. FakeSource.new(None, features=['listen'], created=month_ago, last_poll_attempt=day_and_half_ago).put(), ] resp = cron.application.get_response('/cron/replace_poll_tasks') self.assertEqual(200, resp.status_int) tasks = self.taskqueue_stub.GetTasks('poll') self.assertEqual(1, len(tasks)) self.assert_equals(sources[4].urlsafe(), testutil.get_task_params(tasks[0])['source_key'])
def test_registration_with_user_url(self): """Run through an authorization back and forth with a custom user url provided to the auth mechanism """ encoded_state = urllib.quote_plus( '{"callback":"http://withknown.com/bridgy_callback","feature":"listen",' '"operation":"add","user_url":"https://kylewm.com"}') application = webapp2.WSGIApplication([ ('/fakesource/start', testutil.FakeStartHandler), ('/fakesource/add', testutil.FakeAddHandler), ]) self.expect_webmention_requests_get( 'https://kylewm.com/', response='<html><link rel="webmention" href="/webmention"></html>', verify=False) self.mox.ReplayAll() resp = application.get_response( '/fakesource/start', method='POST', body=urllib.urlencode({ 'feature': 'listen', 'callback': 'http://withknown.com/bridgy_callback', 'user_url': 'https://kylewm.com', })) expected_auth_url = 'http://fake/auth/url?' + urllib.urlencode({ 'redirect_uri': 'http://localhost/fakesource/add?state=' + encoded_state, }) self.assert_equals(302, resp.status_code) self.assert_equals(expected_auth_url, resp.headers['location']) resp = application.get_response( '/fakesource/add?state=' + encoded_state + '&oauth_token=fake-token&oauth_token_secret=fake-secret') self.assert_equals(302, resp.status_code) self.assert_equals( 'http://withknown.com/bridgy_callback?' + urllib.urlencode([ ('result', 'success'), ('key', ndb.Key('FakeSource', '0123456789').urlsafe()), ('user', 'http://localhost/fake/0123456789')]), resp.headers['location']) self.assertEquals( 'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/', resp.headers['Set-Cookie']) source = FakeSource.get_by_id('0123456789') self.assertTrue(source) self.assert_equals('Fake User', source.name) self.assert_equals(['listen'], source.features) self.assert_equals(['https://kylewm.com/', 'http://fakeuser.com/'], source.domain_urls) self.assert_equals(['kylewm.com', 'fakeuser.com'], source.domains)
def test_is_beta_user(self): source = FakeSource.new(self.handler) self.assertFalse(source.is_beta_user()) self.mox.stubs.Set(util, 'BETA_USER_PATHS', set()) self.assertFalse(source.is_beta_user()) self.mox.stubs.Set(util, 'BETA_USER_PATHS', set([source.bridgy_path()])) self.assertTrue(source.is_beta_user())
def test_has_bridgy_webmention_endpoint(self): source = FakeSource.new(None) for endpoint, has in ((None, False), ('http://foo', False ), ('https://brid.gy/webmention/fake', True), ('https://www.brid.gy/webmention/fake', True), ): source.webmention_endpoint = endpoint self.assertEquals(has, source.has_bridgy_webmention_endpoint(), endpoint)
def test_verify_without_webmention_endpoint(self): self.expect_webmention_requests_get( 'http://primary/', 'no webmention endpoint here!', verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://primary/'], domains=['primary']) source.verify() self.assertIsNone(source.webmention_endpoint)
def test_last_activity_id(self): """We should store the last activity id seen and then send it as min_id.""" self.sources[0].set_activities(list(reversed(self.activities))) self.post_task() source = self.sources[0].key.get() self.assertEqual('c', source.last_activity_id) source.last_polled = util.EPOCH source.put() self.mox.StubOutWithMock(FakeSource, 'get_activities_response') FakeSource.get_activities_response( count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True, fetch_shares=True, etag=None, min_id='c', cache=mox.IgnoreArg(), ).AndReturn({'items': []}) self.mox.ReplayAll() self.post_task()
def test_create_new(self): self.assertEqual(0, FakeSource.query().count()) self._test_create_new(features=['listen']) msg = "Added fake (FakeSource). Refresh in a minute to see what we've found!" self.assert_equals({msg}, self.handler.messages) for queue in 'poll', 'poll-now': task_params = testutil.get_task_params(self.taskqueue_stub.GetTasks(queue)[0]) self.assertEqual('1970-01-01-00-00-00', task_params['last_polled'])
def test_verify_without_webmention_endpoint(self): self.expect_requests_get('http://primary/', 'no webmention endpoint here!', verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://primary/'], domains=['primary']) source.verify() self.assertIsNone(source.webmention_endpoint)
def test_create_new_domain_url_path_fails(self): auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'urls': [{'value': 'http://flaky/foo'}]})) self.expect_requests_get('http://flaky', status_code=500) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://flaky/foo'], source.domain_urls) self.assertEquals(['flaky'], source.domains)
def test_get_comment(self): comment_obj = {'objectType': 'comment', 'content': 'qwert'} source = FakeSource.new(None) source.as_source = self.mox.CreateMock(as_source.Source) source.as_source.get_comment('123', activity_id=None, activity_author_id=None ).AndReturn(comment_obj) self.mox.ReplayAll() self.assert_equals(comment_obj, source.get_comment('123'))
def test_create_new_domain_url_path_connection_fails(self): auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'urls': [{'value': 'http://flaky/foo'}]})) self.expect_requests_get('http://flaky').AndRaise( requests.ConnectionError('DNS lookup failed for URL: http://bad/')) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://flaky/foo'], source.domain_urls) self.assertEquals(['flaky'], source.domains)
def test_registration_callback(self): """Run through an authorization back and forth and make sure that the external callback makes it all the way through. """ encoded_state = urllib.quote_plus( '{"callback":"http://withknown.com/bridgy_callback",' '"feature":"listen","operation":"add"}' ) application = webapp2.WSGIApplication( [("/fakesource/start", testutil.FakeStartHandler), ("/fakesource/add", testutil.FakeAddHandler)] ) self.expect_webmention_requests_get( u"http://fakeuser.com/", response='<html><link rel="webmention" href="/webmention"></html>', verify=False ) self.mox.ReplayAll() resp = application.get_response( "/fakesource/start", method="POST", body=urllib.urlencode({"feature": "listen", "callback": "http://withknown.com/bridgy_callback"}), ) expected_auth_url = "http://fake/auth/url?" + urllib.urlencode( {"redirect_uri": "http://localhost/fakesource/add?state=" + encoded_state} ) self.assert_equals(302, resp.status_code) self.assert_equals(expected_auth_url, resp.headers["location"]) resp = application.get_response( "/fakesource/add?state=" + encoded_state + "&oauth_token=fake-token&oauth_token_secret=fake-secret" ) self.assert_equals(302, resp.status_code) self.assert_equals( "http://withknown.com/bridgy_callback?" + urllib.urlencode( [ ("result", "success"), ("key", ndb.Key("FakeSource", "0123456789").urlsafe()), ("user", "http://localhost/fake/0123456789"), ] ), resp.headers["location"], ) self.assertEquals( 'logins="/fake/0123456789?Fake+User"; expires=2001-12-31 00:00:00; Path=/', resp.headers["Set-Cookie"] ) source = FakeSource.get_by_id("0123456789") self.assertTrue(source) self.assert_equals("Fake User", source.name) self.assert_equals(["listen"], source.features)
def test_create_new_dedupes_domains(self): auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'urls': [{'value': 'http://foo'}, {'value': 'https://foo/'}, {'value': 'http://foo/'}, {'value': 'http://foo'}, ]})) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['https://foo/'], source.domain_urls) self.assertEquals(['foo'], source.domains)
def test_etag(self): """If we see an ETag, we should send it with the next get_activities().""" self.sources[0]._set('etag', '"my etag"') self.post_task() source = self.sources[0].key.get() self.assertEqual('"my etag"', source.last_activities_etag) source.last_polled = util.EPOCH source.put() self.mox.StubOutWithMock(FakeSource, 'get_activities_response') FakeSource.get_activities_response( count=mox.IgnoreArg(), fetch_replies=True, fetch_likes=True, fetch_shares=True, etag='"my etag"', min_id='c', cache=mox.IgnoreArg(), ).AndReturn({'items': [], 'etag': '"new etag"'}) self.mox.ReplayAll() self.post_task() source = self.sources[0].key.get() self.assertEqual('"new etag"', source.last_activities_etag)
def test_verify(self): # this requests.get is called by webmention-tools self.expect_webmention_requests_get('http://primary/', """ <html><meta> <link rel="webmention" href="http://web.ment/ion"> </meta></html>""", verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://primary/'], domains=['primary']) source.verify() self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_domain_url_path_fails(self): auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'urls': [{ 'value': 'http://flaky/foo' }]})) self.expect_requests_get('http://flaky', status_code=500) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://flaky/foo'], source.domain_urls) self.assertEquals(['flaky'], source.domains)
def test_verify(self): # this requests.get is called by webmention-tools self.expect_requests_get('http://primary/', """ <html><meta> <link rel="webmention" href="http://web.ment/ion"> </meta></html>""", verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://primary/'], domains=['primary']) source.verify() self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_domain_url_no_root_relme(self): """If a profile URL contains a path, check the root for a rel=me to the path.""" auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': 'http://site/path'})) auth_entity.put() self.expect_requests_get('http://site') self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://site/path'], source.domain_urls) self.assertEquals(['site'], source.domains)
def test_create_new_domain_url_redirects_to_path(self): """If a profile URL is a root that redirects to a path, keep the root.""" auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'url': 'http://site'})) auth_entity.put() self.expect_requests_head('http://site', redirected_url='https://site/path') self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://site/'], source.domain_urls) self.assertEquals(['site'], source.domains)
def test_verify_checks_blacklist(self): self.expect_webmention_requests_get('http://good/', """ <html><meta> <link rel="webmention" href="http://web.ment/ion"> </meta></html>""", verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://bad.app/', 'http://good/'], domains=['bad.app', 'good']) source.verify() self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_create_new_too_many_domains(self): urls = ['http://%s/' % i for i in range(10)] auth_entity = testutil.FakeAuthEntity(id='x', user_json=json.dumps( {'urls': [{'value': u} for u in urls]})) # we should only check the first 5 for url in urls[:models.MAX_AUTHOR_URLS]: self.expect_requests_head(url) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(urls, source.domain_urls) self.assertEquals([str(i) for i in range(10)], source.domains)
def test_put_updates(self): source = FakeSource.new(None) source.put() updates = source.updates = {'status': 'disabled'} try: # check that source.updates is preserved through pre-put hook since some # Source subclasses (e.g. FacebookPage) use it. FakeSource._pre_put_hook = lambda fake: self.assertEquals(updates, fake.updates) Source.put_updates(source) self.assertEquals('disabled', source.key.get().status) finally: del FakeSource._pre_put_hook
def test_create_new_domain_url_path_connection_fails(self): auth_entity = testutil.FakeAuthEntity( id='x', user_json=json.dumps({'urls': [{ 'value': 'http://flaky/foo' }]})) self.expect_requests_get('http://flaky').AndRaise( requests.ConnectionError('DNS lookup failed for URL: http://bad/')) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEquals(['http://flaky/foo'], source.domain_urls) self.assertEquals(['flaky'], source.domains)
def test_verify_checks_blacklist(self): self.expect_webmention_requests_get('http://good/', """ <html><meta> <link rel="webmention" href="http://web.ment/ion"> </meta></html>""", verify=False) self.mox.ReplayAll() source = FakeSource.new( self.handler, features=['webmention'], domain_urls=['http://bad.app/', 'http://good/'], domains=['bad.app', 'good']) source.verify() self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_verify_unicode_characters(self): """Older versions of BS4 had an issue where it would check short HTML documents to make sure the user wasn't accidentally passing a URL, but converting the utf-8 document to ascii caused exceptions in some cases. """ # this requests.get is called by webmention-tools self.expect_webmention_requests_get( 'http://primary/', """\xef\xbb\xbf<html><head> <link rel="webmention" href="http://web.ment/ion"></head> </html>""", verify=False) self.mox.ReplayAll() source = FakeSource.new(self.handler, features=['webmention'], domain_urls=['http://primary/'], domains=['primary']) source.verify() self.assertEquals('http://web.ment/ion', source.webmention_endpoint)
def test_poll_period(self): source = FakeSource.new(None) source.put() self.assertEqual(source.FAST_POLL, source.poll_period()) source.created = datetime.datetime(2000, 1, 1) self.assertEqual(source.SLOW_POLL, source.poll_period()) now = datetime.datetime.now() source.last_webmention_sent = now - datetime.timedelta(days=8) self.assertEqual(source.FAST_POLL * 10, source.poll_period()) source.last_webmention_sent = now self.assertEqual(source.FAST_POLL, source.poll_period()) source.rate_limited = True self.assertEqual(source.RATE_LIMITED_POLL, source.poll_period())
def test_should_refetch(self): source = FakeSource.new(None) # haven't found a synd url yet self.assertFalse(source.should_refetch()) source.last_hfeed_refetch = models.REFETCH_HFEED_TRIGGER # override self.assertTrue(source.should_refetch()) source.last_syndication_url = source.last_hfeed_refetch = testutil.NOW # too soon self.assertFalse(source.should_refetch()) source.last_poll_attempt = testutil.NOW # too soon self.assertFalse(source.should_refetch()) hour = datetime.timedelta(hours=1) source.last_hfeed_refetch -= (Source.FAST_REFETCH + hour) self.assertTrue(source.should_refetch()) source.last_syndication_url -= datetime.timedelta(days=15) # slow refetch self.assertFalse(source.should_refetch()) source.last_hfeed_refetch -= (Source.SLOW_REFETCH + hour) self.assertTrue(source.should_refetch())
def test_create_new_publish(self): """If a source is publish only, we shouldn't insert a poll task.""" FakeSource.create_new(self.handler, features=['publish']) self.assertEqual(0, len(self.taskqueue_stub.GetTasks('poll'))) self.assertEqual(0, len(self.taskqueue_stub.GetTasks('poll-now')))
def test_create_new_unicode_chars(self): """We should handle unusual unicode chars in the source's name ok.""" # the invisible character in the middle is an unusual unicode character FakeSource.create_new(self.handler, name=u'a ✁ b')