def test_transient_errors_except_terminates_on_provided_exceptions(self):
"""
If the failure is of a type provided to :func:`transient_errors_except`,
the function it returns will treat it as terminal (returns False)
"""
can_retry = transient_errors_except(DummyException)
self.assertFalse(can_retry(Failure(DummyException())))
def test_transient_errors_except_terminates_on_provided_exceptions(self):
"""
If the failure is of a type provided to :func:`transient_errors_except`,
the function it returns will treat it as terminal (returns False)
"""
can_retry = transient_errors_except(DummyException)
self.assertFalse(can_retry(Failure(DummyException())))
def test_transient_errors_except_defaults_to_all_transient(self):
"""
If no args are provided to :func:`transient_errors_except`, the
function it returns treats all errors as transient (returns True)
"""
can_retry = transient_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertTrue(can_retry(Failure(exception)))
def test_transient_errors_except_defaults_to_all_transient(self):
"""
If no args are provided to :func:`transient_errors_except`, the
function it returns treats all errors as transient (returns True)
"""
can_retry = transient_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertTrue(can_retry(Failure(exception)))
def add_to_load_balancer(log, endpoint, auth_token, lb_config, ip_address, undo, clock=None):
"""
Add an IP addressed to a load balancer based on the lb_config.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str lb_config: An lb_config dictionary.
:param str ip_address: The IP Address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations onto.
:return: Deferred that fires with the Add Node to load balancer response
as a dict.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path, headers=headers(auth_token),
data=json.dumps({"nodes": [{"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"}]}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(
add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
lb_log.msg('Added to load balancer', node_id=result['nodes'][0]['id'])
undo.push(remove_from_load_balancer,
lb_log,
endpoint,
auth_token,
lb_id,
result['nodes'][0]['id'])
return result
return d.addCallback(treq.json_content).addCallback(when_done)
def _remove_from_clb(log,
endpoint,
auth_token,
loadbalancer_id,
node_id,
clock=None):
"""
Remove a node from a CLB load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone authentication token.
:param str loadbalancer_id: The ID for a Cloud Load Balancer.
:param str node_id: The ID for a node in that Cloud Load Balancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id),
'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content
) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(
lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def remove_from_load_balancer(log, endpoint, auth_token, loadbalancer_id,
node_id, clock=None):
"""
Remove a node from a load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str loadbalancer_id: The ID for a cloud loadbalancer.
:param str node_id: The ID for a node in that cloudloadbalancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id), 'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(
remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=20,
timeout=7200,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 20.
:param int timeout: timeout to poll for the server status in seconds.
Default 7200 (2 hours).
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
time_building = clock.seconds() - start_time
if status == 'ACTIVE':
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
log.msg(
"Server changed to '{status}' in {time_building} seconds",
time_building=time_building,
status=status)
raise UnexpectedServerStatus(server_id, status, 'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id, log=log)
sd.addCallback(check_status)
return sd
timeout_description = ("Waiting for server <{0}> to change from BUILD "
"state to ACTIVE state").format(server_id)
return retry_and_timeout(poll,
timeout,
can_retry=transient_errors_except(
UnexpectedServerStatus, ServerDeleted),
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def add_to_clb(log,
endpoint,
auth_token,
lb_config,
ip_address,
undo,
clock=None):
"""
Add an IP address to a Cloud Load Balancer based on the ``lb_config``.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone auth token.
:param dict lb_config: An ``lb_config`` dictionary.
:param str ip_address: The IP address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations
onto.
:return: Deferred that fires with the load balancer response.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path,
headers=headers(auth_token),
data=json.dumps({
"nodes": [{
"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"
}]
}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
node_id = result['nodes'][0]['id']
lb_log.msg('Added to load balancer', node_id=node_id)
undo.push(_remove_from_clb, lb_log, endpoint, auth_token, lb_id,
node_id)
return result
return d.addCallback(treq.json_content).addCallback(when_done)
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=20,
timeout=7200,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 20.
:param int timeout: timeout to poll for the server status in seconds.
Default 7200 (2 hours).
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
time_building = clock.seconds() - start_time
if status == 'ACTIVE':
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
log.msg("Server changed to '{status}' in {time_building} seconds",
time_building=time_building, status=status)
raise UnexpectedServerStatus(
server_id,
status,
'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id, log=log)
sd.addCallback(check_status)
return sd
timeout_description = ("Waiting for server <{0}> to change from BUILD "
"state to ACTIVE state").format(server_id)
return retry_and_timeout(
poll, timeout,
can_retry=transient_errors_except(UnexpectedServerStatus, ServerDeleted),
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=5,
timeout=3600,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 5.
:param int timeout: timeout to poll for the server status in seconds.
Default 3600 (1 hour)
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
if status == 'ACTIVE':
time_building = clock.seconds() - start_time
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
raise UnexpectedServerStatus(
server_id,
status,
'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id)
sd.addCallback(check_status)
return sd
d = retry_and_timeout(
poll, timeout,
can_retry=transient_errors_except(UnexpectedServerStatus),
next_interval=repeating_interval(interval),
clock=clock)
def on_error(f):
if f.check(CancelledError):
time_building = clock.seconds() - start_time
log.msg(('Server {instance_id} failed to change from BUILD state '
'to ACTIVE within a {timeout} second timeout (it has been '
'{time_building} seconds).'),
timeout=timeout, time_building=time_building)
return f
d.addErrback(on_error)
return d
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=5,
timeout=3600,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 5.
:param int timeout: timeout to poll for the server status in seconds.
Default 3600 (1 hour)
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
if status == 'ACTIVE':
time_building = clock.seconds() - start_time
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
raise UnexpectedServerStatus(server_id, status, 'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id)
sd.addCallback(check_status)
return sd
d = retry_and_timeout(
poll,
timeout,
can_retry=transient_errors_except(UnexpectedServerStatus),
next_interval=repeating_interval(interval),
clock=clock)
def on_error(f):
if f.check(CancelledError):
time_building = clock.seconds() - start_time
log.msg(
('Server {instance_id} failed to change from BUILD state '
'to ACTIVE within a {timeout} second timeout (it has been '
'{time_building} seconds).'),
timeout=timeout,
time_building=time_building)
return f
d.addErrback(on_error)
return d
