def create_node(self,
name=None,
size=None,
image=None,
auth=None,
ex_storage_account=None,
ex_customdata=None,
ex_resource_group=None,
ex_user_name=None,
ex_tags=None,
ex_network=None):
global all_nodes, create_calls, quota
if len(all_nodes) >= quota:
raise BaseHTTPError(503, "Quota exceeded")
else:
return super(QuotaDriver, self).create_node(
name=name,
size=size,
image=image,
auth=auth,
ex_storage_account=ex_storage_account,
ex_customdata=ex_customdata,
ex_resource_group=ex_resource_group,
ex_user_name=ex_user_name,
ex_tags=ex_tags,
ex_network=ex_network)
def create_node(self,
name=None,
size=None,
image=None,
auth=None,
ex_storage_account=None,
ex_customdata=None,
ex_resource_group=None,
ex_user_name=None,
ex_tags=None,
ex_network=None):
global create_calls
create_calls += 1
if create_calls < 2:
raise RateLimitReachedError(429,
"Rate limit exceeded",
headers={'retry-after': '2'})
elif create_calls < 3:
raise BaseHTTPError(429, "Rate limit exceeded",
{'retry-after': '1'})
else:
return super(RetryDriver, self).create_node(
name=name,
size=size,
image=image,
auth=auth,
ex_storage_account=ex_storage_account,
ex_customdata=ex_customdata,
ex_resource_group=ex_resource_group,
ex_user_name=ex_user_name,
ex_tags=ex_tags,
ex_network=ex_network)
def test_known_basehttperror_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "InstanceLimitExceeded"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
self.assertEqual(1, self.cloud_client.post_create_node.call_count)
def test_basehttperror_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(500, "Try again"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
self.setup_actor.ping().get(self.TIMEOUT)
self.assertEqual(1, self.cloud_client.post_create_node.call_count)
def test_instance_exceeded_not_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "InstanceLimitExceeded"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
done = self.FUTURE_CLASS()
self.setup_actor.subscribe(done.set)
done.get(self.TIMEOUT)
self.assertEqual(0, self.cloud_client.post_create_node.call_count)
def test_unknown_basehttperror_not_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "Unknown"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
finished = threading.Event()
self.setup_actor.subscribe(lambda _: finished.set())
assert (finished.wait(self.TIMEOUT))
self.assertEqual(0, self.cloud_client.post_create_node.call_count)
def test_destroy_node__destroy_nic_retries(self, time_sleep_mock):
def error(e, **kwargs):
raise e(**kwargs)
node = self.driver.list_nodes()[0]
err = BaseHTTPError(code=400, message="[NicInUse] Cannot destroy")
with mock.patch.object(self.driver, "ex_destroy_nic") as m:
m.side_effect = [err] * 5 + [True] # 5 errors before a success
ret = self.driver.destroy_node(node)
self.assertTrue(ret)
self.assertEqual(6, m.call_count) # 6th call was a success
m.side_effect = [err] * 10 + [True] # 10 errors before a success
with self.assertRaises(BaseHTTPError):
self.driver.destroy_node(node)
self.assertEqual(10, m.call_count) # try 10 times & fail