def remove_from_load_balancer(log, endpoint, auth_token, loadbalancer_id,
node_id, clock=None):
"""
Remove a node from a load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str loadbalancer_id: The ID for a cloud loadbalancer.
:param str node_id: The ID for a node in that cloudloadbalancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id), 'nodes', str(node_id))
def check_422_deleted(failure):
# A LB being deleted sometimes results in a 422. This function
# unfortunately has to parse the body of the message to see if this is an
# acceptable 422 (if the LB has been deleted or the node has already been
# removed, then 'removing from load balancer' as a task should be
# successful - if the LB is in ERROR, then nothing more can be done to
# it except resetting it - may as well remove the server.)
failure.trap(APIError)
error = failure.value
if error.code == 422:
message = json.loads(error.body)['message']
if ('load balancer is deleted' not in message and
'PENDING_DELETE' not in message):
return failure
lb_log.msg(message)
else:
return failure
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
# Success is 200/202. An LB not being found is 404. A node not being
# found is a 404. But a deleted LB sometimes results in a 422.
d.addCallback(log_on_response_code, lb_log, 'Node to delete does not exist', 404)
d.addCallback(check_success, [200, 202, 404])
d.addCallback(treq.content) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(check_422_deleted)
d.addErrback(log_lb_unexpected_errors, path, lb_log, 'remove_node')
return d
d = retry(
remove,
can_retry=retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def add_to_load_balancer(log, endpoint, auth_token, lb_config, ip_address, undo, clock=None):
"""
Add an IP addressed to a load balancer based on the lb_config.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str lb_config: An lb_config dictionary.
:param str ip_address: The IP Address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations onto.
:return: Deferred that fires with the Add Node to load balancer response
as a dict.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path, headers=headers(auth_token),
data=json.dumps({"nodes": [{"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"}]}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(
add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
lb_log.msg('Added to load balancer', node_id=result['nodes'][0]['id'])
undo.push(remove_from_load_balancer,
lb_log,
endpoint,
auth_token,
lb_id,
result['nodes'][0]['id'])
return result
return d.addCallback(treq.json_content).addCallback(when_done)
def test_random_interval(self):
"""
``random_interval`` returns the different random interval each time it
is called
"""
next_interval = random_interval(5, 10)
intervals = set()
for exception in [DummyException(), NotImplementedError(), ValueError(),
FloatingPointError(), IOError()]:
interval = next_interval(exception)
self.assertTrue(5 <= interval <= 10)
self.assertNotIn(interval, intervals)
intervals.add(interval)
def _remove_from_clb(log,
endpoint,
auth_token,
loadbalancer_id,
node_id,
clock=None):
"""
Remove a node from a CLB load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone authentication token.
:param str loadbalancer_id: The ID for a Cloud Load Balancer.
:param str node_id: The ID for a node in that Cloud Load Balancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id),
'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content
) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(
lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def test_random_interval(self):
"""
``random_interval`` returns the different random interval each time it
is called
"""
next_interval = random_interval(5, 10)
intervals = set()
for exception in [
DummyException(),
NotImplementedError(),
ValueError(),
FloatingPointError(),
IOError()
]:
interval = next_interval(exception)
self.assertTrue(5 <= interval <= 10)
self.assertNotIn(interval, intervals)
intervals.add(interval)
def remove_from_load_balancer(log, endpoint, auth_token, loadbalancer_id,
node_id, clock=None):
"""
Remove a node from a load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str loadbalancer_id: The ID for a cloud loadbalancer.
:param str node_id: The ID for a node in that cloudloadbalancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id), 'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(
remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def add_to_clb(log,
endpoint,
auth_token,
lb_config,
ip_address,
undo,
clock=None):
"""
Add an IP address to a Cloud Load Balancer based on the ``lb_config``.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone auth token.
:param dict lb_config: An ``lb_config`` dictionary.
:param str ip_address: The IP address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations
onto.
:return: Deferred that fires with the load balancer response.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path,
headers=headers(auth_token),
data=json.dumps({
"nodes": [{
"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"
}]
}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
node_id = result['nodes'][0]['id']
lb_log.msg('Added to load balancer', node_id=node_id)
undo.push(_remove_from_clb, lb_log, endpoint, auth_token, lb_id,
node_id)
return result
return d.addCallback(treq.json_content).addCallback(when_done)