def handleProblem(code, resp, msg):
if errback:
errback((resp, msg))
return
if code == 429:
hdrs = resp.hdrs.dict
raise RateLimitException(
'rate limit exceeded',
resp,
msg,
by=hdrs.get('x-ratelimit-by', 'customer'),
limit=hdrs.get('x-ratelimit-limit', 10),
period=hdrs.get('x-ratelimit-period', 1),
remaining=hdrs.get('x-ratelimit-remaining', 100))
elif code == 401:
raise AuthException('unauthorized', resp, msg)
else:
raise ResourceException('server error, status code: %s' % code,
response=resp,
body=msg)
def _onBody(self, body, response, user_callback, data, headers):
self._logHeaders(headers)
self._log.debug("%s %s %s %s" % (response.request.method,
response.request.absoluteURI,
response.code,
data))
if response.code < 200 or response.code >= 300:
if response.code == 429:
raise RateLimitException(
'rate limit exceeded', response, body,
by=response.headers.getRawHeaders('x-ratelimit-by', ['customer'])[0],
limit=response.headers.getRawHeaders('x-ratelimit-limit', [10])[0],
period=response.headers.getRawHeaders('x-ratelimit-period', [1])[0],
remaining=response.headers.getRawHeaders('x-ratelimit-remaining', [100])[0]
)
elif response.code == 401:
raise AuthException('unauthorized',
response,
body)
else:
raise ResourceException('server error', response, body)
if body:
try:
jsonOut = json.loads(body)
except:
raise ResourceException('invalid json in response',
response,
body)
else:
jsonOut = None
if user_callback:
# set these in case callback throws, so we have them for errback
self.response = response
self.body = body
return user_callback(jsonOut, body, response)
else:
return jsonOut
def handleProblem(code, resp, msg):
if errback:
errback((resp, msg))
return
if code == 429:
hdrs = self._get_headers(resp)
raise RateLimitException(
"rate limit exceeded",
resp,
msg,
by=hdrs.get("x-ratelimit-by", "customer"),
limit=hdrs.get("x-ratelimit-limit", 10),
period=hdrs.get("x-ratelimit-period", 1),
remaining=hdrs.get("x-ratelimit-remaining", 100),
)
elif code == 401:
raise AuthException("unauthorized", resp, msg)
else:
raise ResourceException(
"server error, status code: %s" % code,
response=resp,
body=msg,
)
def test_sync(self, zone_retrieve_mock, zone_create_mock,
record_retrieve_mock, record_create_mock, record_update_mock,
record_delete_mock):
provider = Ns1Provider('test', 'api-key')
desired = Zone('unit.tests.', [])
for r in self.expected:
desired.add_record(r)
plan = provider.plan(desired)
# everything except the root NS
expected_n = len(self.expected) - 1
self.assertEquals(expected_n, len(plan.changes))
self.assertTrue(plan.exists)
# Fails, general error
zone_retrieve_mock.reset_mock()
zone_create_mock.reset_mock()
zone_retrieve_mock.side_effect = ResourceException('boom')
with self.assertRaises(ResourceException) as ctx:
provider.apply(plan)
self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception)
# Fails, bad auth
zone_retrieve_mock.reset_mock()
zone_create_mock.reset_mock()
zone_retrieve_mock.side_effect = \
ResourceException('server error: zone not found')
zone_create_mock.side_effect = AuthException('unauthorized')
with self.assertRaises(AuthException) as ctx:
provider.apply(plan)
self.assertEquals(zone_create_mock.side_effect, ctx.exception)
# non-existent zone, create
zone_retrieve_mock.reset_mock()
zone_create_mock.reset_mock()
zone_retrieve_mock.side_effect = \
ResourceException('server error: zone not found')
zone_create_mock.side_effect = ['foo']
# Test out the create rate-limit handling, then 9 successes
record_create_mock.side_effect = [
RateLimitException('boo', period=0),
] + ([None] * 9)
got_n = provider.apply(plan)
self.assertEquals(expected_n, got_n)
# Zone was created
zone_create_mock.assert_has_calls([call('unit.tests')])
# Checking that we got some of the expected records too
record_create_mock.assert_has_calls([
call('unit.tests',
'unit.tests',
'A',
answers=[{
'answer': ['1.2.3.4'],
'meta': {}
}],
filters=[],
ttl=32),
call('unit.tests',
'unit.tests',
'CAA',
answers=[(0, 'issue', 'ca.unit.tests')],
ttl=40),
call('unit.tests',
'unit.tests',
'MX',
answers=[(10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.')],
ttl=35),
])
# Update & delete
zone_retrieve_mock.reset_mock()
zone_create_mock.reset_mock()
ns1_zone = {
'records':
self.ns1_records + [
{
'type': 'A',
'ttl': 42,
'short_answers': ['9.9.9.9'],
'domain': 'delete-me.unit.tests.',
},
{
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"short_answers": [
'1.1.1.1',
'1.2.3.4',
'2.3.4.5',
'3.4.5.6',
'4.5.6.7',
],
'tier':
3, # This flags it as advacned, full load required
'ttl':
34,
}
],
}
ns1_zone['records'][0]['short_answers'][0] = '2.2.2.2'
record_retrieve_mock.side_effect = [{
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"answers": [
{
'answer': ['1.1.1.1'],
'meta': {}
},
{
'answer': ['1.2.3.4'],
'meta': {
'ca_province': ['ON']
}
},
{
'answer': ['2.3.4.5'],
'meta': {
'us_state': ['NY']
}
},
{
'answer': ['3.4.5.6'],
'meta': {
'country': ['US']
}
},
{
'answer': ['4.5.6.7'],
'meta': {
'iso_region_code': ['NA-US-WA']
}
},
],
'tier':
3,
'ttl':
34,
}]
zone_retrieve_mock.side_effect = [ns1_zone, ns1_zone]
plan = provider.plan(desired)
self.assertEquals(3, len(plan.changes))
# Shouldn't rely on order so just count classes
classes = defaultdict(lambda: 0)
for change in plan.changes:
classes[change.__class__] += 1
self.assertEquals(1, classes[Delete])
self.assertEquals(2, classes[Update])
record_update_mock.side_effect = [
RateLimitException('one', period=0),
None,
None,
]
record_delete_mock.side_effect = [
RateLimitException('two', period=0),
None,
None,
]
got_n = provider.apply(plan)
self.assertEquals(3, got_n)
record_update_mock.assert_has_calls([
call('unit.tests',
'unit.tests',
'A',
answers=[{
'answer': ['1.2.3.4'],
'meta': {}
}],
filters=[],
ttl=32),
call('unit.tests',
'unit.tests',
'A',
answers=[{
'answer': ['1.2.3.4'],
'meta': {}
}],
filters=[],
ttl=32),
call('unit.tests',
'geo.unit.tests',
'A',
answers=[{
'answer': ['101.102.103.104'],
'meta': {}
}, {
'answer': ['101.102.103.105'],
'meta': {}
}, {
'answer': ['201.202.203.204'],
'meta': {
'iso_region_code': ['NA-US-NY']
}
}],
filters=[{
'filter': 'shuffle',
'config': {}
}, {
'filter': 'geotarget_country',
'config': {}
}, {
'filter': 'select_first_n',
'config': {
'N': 1
}
}],
ttl=34)
])
def test_populate(self, zone_retrieve_mock):
provider = Ns1Provider('test', 'api-key')
# Bad auth
zone_retrieve_mock.side_effect = AuthException('unauthorized')
zone = Zone('unit.tests.', [])
with self.assertRaises(AuthException) as ctx:
provider.populate(zone)
self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception)
# General error
zone_retrieve_mock.reset_mock()
zone_retrieve_mock.side_effect = ResourceException('boom')
zone = Zone('unit.tests.', [])
with self.assertRaises(ResourceException) as ctx:
provider.populate(zone)
self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception)
self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
# Non-existent zone doesn't populate anything
zone_retrieve_mock.reset_mock()
zone_retrieve_mock.side_effect = \
ResourceException('server error: zone not found')
zone = Zone('unit.tests.', [])
exists = provider.populate(zone)
self.assertEquals(set(), zone.records)
self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
self.assertFalse(exists)
# Existing zone w/o records
zone_retrieve_mock.reset_mock()
ns1_zone = {
'records': [{
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"answers": [
{
'answer': ['1.1.1.1'],
'meta': {}
},
{
'answer': ['1.2.3.4'],
'meta': {
'ca_province': ['ON']
}
},
{
'answer': ['2.3.4.5'],
'meta': {
'us_state': ['NY']
}
},
{
'answer': ['3.4.5.6'],
'meta': {
'country': ['US']
}
},
{
'answer': ['4.5.6.7'],
'meta': {
'iso_region_code': ['NA-US-WA']
}
},
],
'ttl':
34,
}],
}
zone_retrieve_mock.side_effect = [ns1_zone]
zone = Zone('unit.tests.', [])
provider.populate(zone)
self.assertEquals(1, len(zone.records))
self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
# Existing zone w/records
zone_retrieve_mock.reset_mock()
ns1_zone = {
'records':
self.ns1_records + [{
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"answers": [
{
'answer': ['1.1.1.1'],
'meta': {}
},
{
'answer': ['1.2.3.4'],
'meta': {
'ca_province': ['ON']
}
},
{
'answer': ['2.3.4.5'],
'meta': {
'us_state': ['NY']
}
},
{
'answer': ['3.4.5.6'],
'meta': {
'country': ['US']
}
},
{
'answer': ['4.5.6.7'],
'meta': {
'iso_region_code': ['NA-US-WA']
}
},
],
'ttl':
34,
}],
}
zone_retrieve_mock.side_effect = [ns1_zone]
zone = Zone('unit.tests.', [])
provider.populate(zone)
self.assertEquals(self.expected, zone.records)
self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
# Test skipping unsupported record type
zone_retrieve_mock.reset_mock()
ns1_zone = {
'records':
self.ns1_records + [{
'type': 'UNSUPPORTED',
'ttl': 42,
'short_answers': ['unsupported'],
'domain': 'unsupported.unit.tests.',
}, {
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"answers": [
{
'answer': ['1.1.1.1'],
'meta': {}
},
{
'answer': ['1.2.3.4'],
'meta': {
'ca_province': ['ON']
}
},
{
'answer': ['2.3.4.5'],
'meta': {
'us_state': ['NY']
}
},
{
'answer': ['3.4.5.6'],
'meta': {
'country': ['US']
}
},
{
'answer': ['4.5.6.7'],
'meta': {
'iso_region_code': ['NA-US-WA']
}
},
],
'ttl':
34,
}],
}
zone_retrieve_mock.side_effect = [ns1_zone]
zone = Zone('unit.tests.', [])
provider.populate(zone)
self.assertEquals(self.expected, zone.records)
self.assertEquals(('unit.tests', ), zone_retrieve_mock.call_args[0])
def test_sync(self, load_mock, create_mock):
provider = Ns1Provider('test', 'api-key')
desired = Zone('unit.tests.', [])
for r in self.expected:
desired.add_record(r)
plan = provider.plan(desired)
# everything except the root NS
expected_n = len(self.expected) - 1
self.assertEquals(expected_n, len(plan.changes))
self.assertTrue(plan.exists)
# Fails, general error
load_mock.reset_mock()
create_mock.reset_mock()
load_mock.side_effect = ResourceException('boom')
with self.assertRaises(ResourceException) as ctx:
provider.apply(plan)
self.assertEquals(load_mock.side_effect, ctx.exception)
# Fails, bad auth
load_mock.reset_mock()
create_mock.reset_mock()
load_mock.side_effect = \
ResourceException('server error: zone not found')
create_mock.side_effect = AuthException('unauthorized')
with self.assertRaises(AuthException) as ctx:
provider.apply(plan)
self.assertEquals(create_mock.side_effect, ctx.exception)
# non-existent zone, create
load_mock.reset_mock()
create_mock.reset_mock()
load_mock.side_effect = \
ResourceException('server error: zone not found')
# ugh, need a mock zone with a mock prop since we're using getattr, we
# can actually control side effects on `meth` with that.
mock_zone = Mock()
mock_zone.add_SRV = Mock()
mock_zone.add_SRV.side_effect = [
RateLimitException('boo', period=0),
None,
]
create_mock.side_effect = [mock_zone]
got_n = provider.apply(plan)
self.assertEquals(expected_n, got_n)
# Update & delete
load_mock.reset_mock()
create_mock.reset_mock()
nsone_zone = DummyZone(self.nsone_records +
[{
'type': 'A',
'ttl': 42,
'short_answers': ['9.9.9.9'],
'domain': 'delete-me.unit.tests.',
}])
nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2'
nsone_zone.loadRecord = Mock()
zone_search = Mock()
zone_search.return_value = [
{
"domain":
"geo.unit.tests",
"zone":
"unit.tests",
"type":
"A",
"answers": [
{
'answer': ['1.1.1.1'],
'meta': {}
},
{
'answer': ['1.2.3.4'],
'meta': {
'ca_province': ['ON']
}
},
{
'answer': ['2.3.4.5'],
'meta': {
'us_state': ['NY']
}
},
{
'answer': ['3.4.5.6'],
'meta': {
'country': ['US']
}
},
{
'answer': ['4.5.6.7'],
'meta': {
'iso_region_code': ['NA-US-WA']
}
},
],
'ttl':
34,
},
]
nsone_zone.search = zone_search
load_mock.side_effect = [nsone_zone, nsone_zone]
plan = provider.plan(desired)
self.assertEquals(3, len(plan.changes))
self.assertIsInstance(plan.changes[0], Update)
self.assertIsInstance(plan.changes[2], Delete)
# ugh, we need a mock record that can be returned from loadRecord for
# the update and delete targets, we can add our side effects to that to
# trigger rate limit handling
mock_record = Mock()
mock_record.update.side_effect = [
RateLimitException('one', period=0),
None,
None,
]
mock_record.delete.side_effect = [
RateLimitException('two', period=0),
None,
None,
]
nsone_zone.loadRecord.side_effect = [
mock_record, mock_record, mock_record
]
got_n = provider.apply(plan)
self.assertEquals(3, got_n)
nsone_zone.loadRecord.assert_has_calls([
call('unit.tests', u'A'),
call('geo', u'A'),
call('delete-me', u'A'),
])
mock_record.assert_has_calls([
call.update(answers=[{
'answer': [u'1.2.3.4'],
'meta': {}
}],
filters=[],
ttl=32),
call.update(answers=[{
u'answer': [u'1.2.3.4'],
u'meta': {}
}],
filters=[],
ttl=32),
call.update(answers=[
{
u'answer': [u'101.102.103.104'],
u'meta': {}
},
{
u'answer': [u'101.102.103.105'],
u'meta': {}
},
{
u'answer': [u'201.202.203.204'],
u'meta': {
u'iso_region_code': [u'NA-US-NY']
},
},
],
filters=[
{
u'filter': u'shuffle',
u'config': {}
},
{
u'filter': u'geotarget_country',
u'config': {}
},
{
u'filter': u'select_first_n',
u'config': {
u'N': 1
}
},
],
ttl=34),
call.delete(),
call.delete()
])
def send(
self,
method,
url,
headers=None,
data=None,
params=None,
files=None,
callback=None,
errback=None,
):
self._logHeaders(headers)
resp = self.REQ_MAP[method](
url,
headers=headers,
verify=self._verify,
data=data,
files=files,
params=params,
timeout=self._timeout,
)
response_headers = resp.headers
rate_limit_headers = self._rateLimitHeaders(response_headers)
self._rate_limit_func(rate_limit_headers)
if resp.status_code < 200 or resp.status_code >= 300:
if errback:
errback(resp)
return
else:
if resp.status_code == 429:
raise RateLimitException(
"rate limit exceeded",
resp,
resp.text,
by=rate_limit_headers["by"],
limit=rate_limit_headers["limit"],
period=rate_limit_headers["period"],
remaining=rate_limit_headers["remaining"],
)
elif resp.status_code == 401:
raise AuthException("unauthorized", resp, resp.text)
else:
raise ResourceException("server error", resp, resp.text)
# TODO make sure json is valid if a body is returned
if resp.text:
try:
jsonOut = resp.json()
except ValueError:
if errback:
errback(resp)
return
else:
raise ResourceException("invalid json in response", resp,
resp.text)
else:
jsonOut = None
if callback:
return callback(jsonOut)
else:
return jsonOut