def handleProblem(code, resp, msg): if errback: errback((resp, msg)) return if code == 429: hdrs = resp.hdrs.dict raise RateLimitException( 'rate limit exceeded', resp, msg, by=hdrs.get('x-ratelimit-by', 'customer'), limit=hdrs.get('x-ratelimit-limit', 10), period=hdrs.get('x-ratelimit-period', 1), remaining=hdrs.get('x-ratelimit-remaining', 100)) elif code == 401: raise AuthException('unauthorized', resp, msg) else: raise ResourceException('server error, status code: %s' % code, response=resp, body=msg)
def _onBody(self, body, response, user_callback, data, headers): self._logHeaders(headers) self._log.debug("%s %s %s %s" % (response.request.method, response.request.absoluteURI, response.code, data)) if response.code < 200 or response.code >= 300: if response.code == 429: raise RateLimitException( 'rate limit exceeded', response, body, by=response.headers.getRawHeaders('x-ratelimit-by', ['customer'])[0], limit=response.headers.getRawHeaders('x-ratelimit-limit', [10])[0], period=response.headers.getRawHeaders('x-ratelimit-period', [1])[0], remaining=response.headers.getRawHeaders('x-ratelimit-remaining', [100])[0] ) elif response.code == 401: raise AuthException('unauthorized', response, body) else: raise ResourceException('server error', response, body) if body: try: jsonOut = json.loads(body) except: raise ResourceException('invalid json in response', response, body) else: jsonOut = None if user_callback: # set these in case callback throws, so we have them for errback self.response = response self.body = body return user_callback(jsonOut, body, response) else: return jsonOut
def handleProblem(code, resp, msg): if errback: errback((resp, msg)) return if code == 429: hdrs = self._get_headers(resp) raise RateLimitException( "rate limit exceeded", resp, msg, by=hdrs.get("x-ratelimit-by", "customer"), limit=hdrs.get("x-ratelimit-limit", 10), period=hdrs.get("x-ratelimit-period", 1), remaining=hdrs.get("x-ratelimit-remaining", 100), ) elif code == 401: raise AuthException("unauthorized", resp, msg) else: raise ResourceException( "server error, status code: %s" % code, response=resp, body=msg, )
def test_sync(self, zone_retrieve_mock, zone_create_mock, record_retrieve_mock, record_create_mock, record_update_mock, record_delete_mock): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) for r in self.expected: desired.add_record(r) plan = provider.plan(desired) # everything except the root NS expected_n = len(self.expected) - 1 self.assertEquals(expected_n, len(plan.changes)) self.assertTrue(plan.exists) # Fails, general error zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone_create_mock.side_effect = AuthException('unauthorized') with self.assertRaises(AuthException) as ctx: provider.apply(plan) self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone_create_mock.side_effect = ['foo'] # Test out the create rate-limit handling, then 9 successes record_create_mock.side_effect = [ RateLimitException('boo', period=0), ] + ([None] * 9) got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) # Zone was created zone_create_mock.assert_has_calls([call('unit.tests')]) # Checking that we got some of the expected records too record_create_mock.assert_has_calls([ call('unit.tests', 'unit.tests', 'A', answers=[{ 'answer': ['1.2.3.4'], 'meta': {} }], filters=[], ttl=32), call('unit.tests', 'unit.tests', 'CAA', answers=[(0, 'issue', 'ca.unit.tests')], ttl=40), call('unit.tests', 'unit.tests', 'MX', answers=[(10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.')], ttl=35), ]) # Update & delete zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() ns1_zone = { 'records': self.ns1_records + [ { 'type': 'A', 'ttl': 42, 'short_answers': ['9.9.9.9'], 'domain': 'delete-me.unit.tests.', }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "short_answers": [ '1.1.1.1', '1.2.3.4', '2.3.4.5', '3.4.5.6', '4.5.6.7', ], 'tier': 3, # This flags it as advacned, full load required 'ttl': 34, } ], } ns1_zone['records'][0]['short_answers'][0] = '2.2.2.2' record_retrieve_mock.side_effect = [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "answers": [ { 'answer': ['1.1.1.1'], 'meta': {} }, { 'answer': ['1.2.3.4'], 'meta': { 'ca_province': ['ON'] } }, { 'answer': ['2.3.4.5'], 'meta': { 'us_state': ['NY'] } }, { 'answer': ['3.4.5.6'], 'meta': { 'country': ['US'] } }, { 'answer': ['4.5.6.7'], 'meta': { 'iso_region_code': ['NA-US-WA'] } }, ], 'tier': 3, 'ttl': 34, }] zone_retrieve_mock.side_effect = [ns1_zone, ns1_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) # Shouldn't rely on order so just count classes classes = defaultdict(lambda: 0) for change in plan.changes: classes[change.__class__] += 1 self.assertEquals(1, classes[Delete]) self.assertEquals(2, classes[Update]) record_update_mock.side_effect = [ RateLimitException('one', period=0), None, None, ] record_delete_mock.side_effect = [ RateLimitException('two', period=0), None, None, ] got_n = provider.apply(plan) self.assertEquals(3, got_n) record_update_mock.assert_has_calls([ call('unit.tests', 'unit.tests', 'A', answers=[{ 'answer': ['1.2.3.4'], 'meta': {} }], filters=[], ttl=32), call('unit.tests', 'unit.tests', 'A', answers=[{ 'answer': ['1.2.3.4'], 'meta': {} }], filters=[], ttl=32), call('unit.tests', 'geo.unit.tests', 'A', answers=[{ 'answer': ['101.102.103.104'], 'meta': {} }, { 'answer': ['101.102.103.105'], 'meta': {} }, { 'answer': ['201.202.203.204'], 'meta': { 'iso_region_code': ['NA-US-NY'] } }], filters=[{ 'filter': 'shuffle', 'config': {} }, { 'filter': 'geotarget_country', 'config': {} }, { 'filter': 'select_first_n', 'config': { 'N': 1 } }], ttl=34) ])
def test_populate(self, zone_retrieve_mock): provider = Ns1Provider('test', 'api-key') # Bad auth zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: provider.populate(zone) self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: provider.populate(zone) self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) exists = provider.populate(zone) self.assertEquals(set(), zone.records) self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0]) self.assertFalse(exists) # Existing zone w/o records zone_retrieve_mock.reset_mock() ns1_zone = { 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "answers": [ { 'answer': ['1.1.1.1'], 'meta': {} }, { 'answer': ['1.2.3.4'], 'meta': { 'ca_province': ['ON'] } }, { 'answer': ['2.3.4.5'], 'meta': { 'us_state': ['NY'] } }, { 'answer': ['3.4.5.6'], 'meta': { 'country': ['US'] } }, { 'answer': ['4.5.6.7'], 'meta': { 'iso_region_code': ['NA-US-WA'] } }, ], 'ttl': 34, }], } zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0]) # Existing zone w/records zone_retrieve_mock.reset_mock() ns1_zone = { 'records': self.ns1_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "answers": [ { 'answer': ['1.1.1.1'], 'meta': {} }, { 'answer': ['1.2.3.4'], 'meta': { 'ca_province': ['ON'] } }, { 'answer': ['2.3.4.5'], 'meta': { 'us_state': ['NY'] } }, { 'answer': ['3.4.5.6'], 'meta': { 'country': ['US'] } }, { 'answer': ['4.5.6.7'], 'meta': { 'iso_region_code': ['NA-US-WA'] } }, ], 'ttl': 34, }], } zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0]) # Test skipping unsupported record type zone_retrieve_mock.reset_mock() ns1_zone = { 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', 'ttl': 42, 'short_answers': ['unsupported'], 'domain': 'unsupported.unit.tests.', }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "answers": [ { 'answer': ['1.1.1.1'], 'meta': {} }, { 'answer': ['1.2.3.4'], 'meta': { 'ca_province': ['ON'] } }, { 'answer': ['2.3.4.5'], 'meta': { 'us_state': ['NY'] } }, { 'answer': ['3.4.5.6'], 'meta': { 'country': ['US'] } }, { 'answer': ['4.5.6.7'], 'meta': { 'iso_region_code': ['NA-US-WA'] } }, ], 'ttl': 34, }], } zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
def test_sync(self, load_mock, create_mock): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) for r in self.expected: desired.add_record(r) plan = provider.plan(desired) # everything except the root NS expected_n = len(self.expected) - 1 self.assertEquals(expected_n, len(plan.changes)) self.assertTrue(plan.exists) # Fails, general error load_mock.reset_mock() create_mock.reset_mock() load_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) self.assertEquals(load_mock.side_effect, ctx.exception) # Fails, bad auth load_mock.reset_mock() create_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') create_mock.side_effect = AuthException('unauthorized') with self.assertRaises(AuthException) as ctx: provider.apply(plan) self.assertEquals(create_mock.side_effect, ctx.exception) # non-existent zone, create load_mock.reset_mock() create_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') # ugh, need a mock zone with a mock prop since we're using getattr, we # can actually control side effects on `meth` with that. mock_zone = Mock() mock_zone.add_SRV = Mock() mock_zone.add_SRV.side_effect = [ RateLimitException('boo', period=0), None, ] create_mock.side_effect = [mock_zone] got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) # Update & delete load_mock.reset_mock() create_mock.reset_mock() nsone_zone = DummyZone(self.nsone_records + [{ 'type': 'A', 'ttl': 42, 'short_answers': ['9.9.9.9'], 'domain': 'delete-me.unit.tests.', }]) nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' nsone_zone.loadRecord = Mock() zone_search = Mock() zone_search.return_value = [ { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", "answers": [ { 'answer': ['1.1.1.1'], 'meta': {} }, { 'answer': ['1.2.3.4'], 'meta': { 'ca_province': ['ON'] } }, { 'answer': ['2.3.4.5'], 'meta': { 'us_state': ['NY'] } }, { 'answer': ['3.4.5.6'], 'meta': { 'country': ['US'] } }, { 'answer': ['4.5.6.7'], 'meta': { 'iso_region_code': ['NA-US-WA'] } }, ], 'ttl': 34, }, ] nsone_zone.search = zone_search load_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) self.assertIsInstance(plan.changes[0], Update) self.assertIsInstance(plan.changes[2], Delete) # ugh, we need a mock record that can be returned from loadRecord for # the update and delete targets, we can add our side effects to that to # trigger rate limit handling mock_record = Mock() mock_record.update.side_effect = [ RateLimitException('one', period=0), None, None, ] mock_record.delete.side_effect = [ RateLimitException('two', period=0), None, None, ] nsone_zone.loadRecord.side_effect = [ mock_record, mock_record, mock_record ] got_n = provider.apply(plan) self.assertEquals(3, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), call('geo', u'A'), call('delete-me', u'A'), ]) mock_record.assert_has_calls([ call.update(answers=[{ 'answer': [u'1.2.3.4'], 'meta': {} }], filters=[], ttl=32), call.update(answers=[{ u'answer': [u'1.2.3.4'], u'meta': {} }], filters=[], ttl=32), call.update(answers=[ { u'answer': [u'101.102.103.104'], u'meta': {} }, { u'answer': [u'101.102.103.105'], u'meta': {} }, { u'answer': [u'201.202.203.204'], u'meta': { u'iso_region_code': [u'NA-US-NY'] }, }, ], filters=[ { u'filter': u'shuffle', u'config': {} }, { u'filter': u'geotarget_country', u'config': {} }, { u'filter': u'select_first_n', u'config': { u'N': 1 } }, ], ttl=34), call.delete(), call.delete() ])
def send( self, method, url, headers=None, data=None, params=None, files=None, callback=None, errback=None, ): self._logHeaders(headers) resp = self.REQ_MAP[method]( url, headers=headers, verify=self._verify, data=data, files=files, params=params, timeout=self._timeout, ) response_headers = resp.headers rate_limit_headers = self._rateLimitHeaders(response_headers) self._rate_limit_func(rate_limit_headers) if resp.status_code < 200 or resp.status_code >= 300: if errback: errback(resp) return else: if resp.status_code == 429: raise RateLimitException( "rate limit exceeded", resp, resp.text, by=rate_limit_headers["by"], limit=rate_limit_headers["limit"], period=rate_limit_headers["period"], remaining=rate_limit_headers["remaining"], ) elif resp.status_code == 401: raise AuthException("unauthorized", resp, resp.text) else: raise ResourceException("server error", resp, resp.text) # TODO make sure json is valid if a body is returned if resp.text: try: jsonOut = resp.json() except ValueError: if errback: errback(resp) return else: raise ResourceException("invalid json in response", resp, resp.text) else: jsonOut = None if callback: return callback(jsonOut) else: return jsonOut