def _do_check_resource(
self, cnxt, current_traversal, tmpl, resource_data, is_update, rsrc, stack, adopt_stack_data
):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, resource_data, self.engine_id, stack)
except exception.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info("Replacing resource with new id %s", new_res_id)
rpc_data = sync_point.serialize_input_data(resource_data)
self._rpc_client.check_resource(
cnxt, new_res_id, current_traversal, rpc_data, is_update, adopt_stack_data
)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, resource_data, self.engine_id, stack.time_remaining())
return True
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(resource_data)
self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, rpc_data, is_update, adopt_stack_data)
except exception.ResourceFailure as ex:
reason = "Resource %s failed: %s" % (rsrc.action, six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id, stack, reason)
except scheduler.Timeout:
# reload the stack to verify current traversal
stack = parser.Stack.load(cnxt, stack_id=stack.id)
if stack.current_traversal != current_traversal:
return
self._handle_stack_timeout(cnxt, stack)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, resource_data,
self.engine_id,
stack, self.msg_queue)
except resource.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info("Replacing resource with new id %s",
new_res_id)
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt,
new_res_id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, resource_data,
self.engine_id,
stack.time_remaining(), self.msg_queue)
return True
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(self.input_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
# reload the stack to verify current traversal
stack = parser.Stack.load(cnxt, stack_id=stack.id)
if stack.current_traversal != current_traversal:
return
self._handle_stack_timeout(cnxt, stack)
except CancelOperation:
pass
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, resource_data,
self.engine_id,
stack)
except resource.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info(_LI("Replacing resource with new id %s"),
new_res_id)
rpc_data = sync_point.serialize_input_data(resource_data)
self._rpc_client.check_resource(cnxt,
new_res_id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, resource_data,
self.engine_id, stack.time_remaining())
return True
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(resource_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
# reload the stack to verify current traversal
stack = parser.Stack.load(cnxt, stack_id=stack.id)
if stack.current_traversal != current_traversal:
return
self._handle_stack_timeout(cnxt, stack)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, data,
is_update, rsrc, stack_id):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, data, self.engine_id)
except resource.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info("Replacing resource with new id %s", new_res_id)
data = sync_point.serialize_input_data(data)
self._rpc_client.check_resource(cnxt, new_res_id,
current_traversal, data,
is_update)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id)
return True
except resource.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
self._rpc_client.check_resource(cnxt, rsrc.id,
current_traversal, data,
is_update)
except exception.ResourceFailure as ex:
reason = 'Resource %s failed: %s' % (rsrc.action,
six.text_type(ex))
self._handle_resource_failure(cnxt, stack_id, current_traversal,
reason)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, data,
is_update, rsrc, stack_id):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, data, self.engine_id)
except resource.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info("Replacing resource with new id %s", new_res_id)
data = sync_point.serialize_input_data(data)
self._rpc_client.check_resource(cnxt,
new_res_id,
current_traversal,
data, is_update)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id)
return True
except resource.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
data, is_update)
except exception.ResourceFailure as ex:
reason = 'Resource %s failed: %s' % (rsrc.action,
six.text_type(ex))
self._handle_resource_failure(
cnxt, stack_id, current_traversal, reason)
return False
def test_requires(self, mock_load, mock_cru, mock_crc, mock_pcr, mock_csc):
mock_load.return_value = self.resource, self.stack, self.stack
res_data = {(1, True): {u'id': 5, u'name': 'A', 'attrs': {}},
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal,
sync_point.serialize_input_data(res_data),
self.is_update, {})
mock_cru.assert_called_once_with(
self.resource, self.resource.stack.t.id,
{5, 3}, self.worker.engine_id,
self.stack, mock.ANY)
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, resource_data,
self.engine_id,
stack, self.msg_queue)
except resource.UpdateReplace:
self._handle_resource_replacement(cnxt, current_traversal,
tmpl.id,
rsrc, stack,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, resource_data,
self.engine_id,
stack.time_remaining(), self.msg_queue)
return True
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(self.input_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, u'Timed out')
except CancelOperation as ex:
# Stack is already marked FAILED, so we just need to retrigger
# in case a new traversal has started and is waiting on us.
self._retrigger_new_traversal(cnxt, current_traversal, is_update,
stack.id, rsrc.id)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
prev_template_id = rsrc.current_template_id
try:
if is_update:
requires = set(d.primary_key for d in resource_data.values()
if d is not None)
try:
check_resource_update(rsrc, tmpl.id, requires,
self.engine_id,
stack, self.msg_queue)
except resource.UpdateReplace:
self._handle_resource_replacement(cnxt, current_traversal,
tmpl.id, requires,
rsrc, stack,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, self.engine_id,
stack.time_remaining(), self.msg_queue)
return True
except exception.UpdateInProgress:
LOG.debug('Waiting for existing update to unlock resource %s',
rsrc.id)
if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id):
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
else:
rsrc.handle_preempt()
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, u'Timed out')
except CancelOperation as ex:
# Stack is already marked FAILED, so we just need to retrigger
# in case a new traversal has started and is waiting on us.
self._retrigger_new_traversal(cnxt, current_traversal, is_update,
stack.id, rsrc.id)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
try:
if is_update:
try:
check_resource_update(rsrc, tmpl.id, resource_data,
self.engine_id,
stack, self.msg_queue)
except resource.UpdateReplace:
self._handle_resource_replacement(cnxt, current_traversal,
tmpl.id,
rsrc, stack,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, resource_data,
self.engine_id,
stack.time_remaining(), self.msg_queue)
return True
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(self.input_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, u'Timed out')
except CancelOperation as ex:
# Stack is already marked FAILED, so we just need to retrigger
# in case a new traversal has started and is waiting on us.
self._retrigger_new_traversal(cnxt, current_traversal, is_update,
stack.id, rsrc.id)
return False
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
is_update, rsrc, stack, adopt_stack_data):
prev_template_id = rsrc.current_template_id
try:
if is_update:
requires = set(d.primary_key for d in resource_data.values()
if d is not None)
try:
check_resource_update(rsrc, tmpl.id, requires,
self.engine_id,
stack, self.msg_queue)
except resource.UpdateReplace:
self._handle_resource_replacement(cnxt, current_traversal,
tmpl.id, requires,
rsrc, stack,
adopt_stack_data)
return False
else:
check_resource_cleanup(rsrc, tmpl.id, self.engine_id,
stack.time_remaining(), self.msg_queue)
return True
except exception.UpdateInProgress:
if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id):
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,
rpc_data, is_update,
adopt_stack_data)
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, u'Timed out')
except CancelOperation as ex:
# Stack is already marked FAILED, so we just need to retrigger
# in case a new traversal has started and is waiting on us.
self._retrigger_new_traversal(cnxt, current_traversal, is_update,
stack.id, rsrc.id)
return False
def _handle_resource_replacement(self, cnxt, current_traversal,
new_tmpl_id, rsrc, stack,
adopt_stack_data):
"""Create a replacement resource and trigger a check on it."""
try:
new_res_id = rsrc.make_replacement(new_tmpl_id)
except exception.UpdateInProgress:
LOG.info("No replacement created - "
"resource already locked by new traversal")
return
if new_res_id is None:
LOG.info("No replacement created - "
"new traversal already in progress")
self._retrigger_new_traversal(cnxt, current_traversal, True,
stack.id, rsrc.id)
return
LOG.info("Replacing resource with new id %s", new_res_id)
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt, new_res_id, current_traversal,
rpc_data, True, adopt_stack_data)
def _handle_resource_replacement(self, cnxt,
current_traversal, new_tmpl_id, requires,
rsrc, stack, adopt_stack_data):
"""Create a replacement resource and trigger a check on it."""
try:
new_res_id = rsrc.make_replacement(new_tmpl_id, requires)
except exception.UpdateInProgress:
LOG.info("No replacement created - "
"resource already locked by new traversal")
return
if new_res_id is None:
LOG.info("No replacement created - "
"new traversal already in progress")
self._retrigger_new_traversal(cnxt, current_traversal, True,
stack.id, rsrc.id)
return
LOG.info("Replacing resource with new id %s", new_res_id)
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt,
new_res_id,
current_traversal,
rpc_data, True,
adopt_stack_data)
def test_serialize_input_data(self):
res = sync_point.serialize_input_data({(3, 8): None})
self.assertEqual({'input_data': [[[3, 8], None]]}, res)
def test_serialize_input_data(self):
res = sync_point.serialize_input_data({(3, 8): None})
self.assertEqual({'input_data': {u'tuple:(3, 8)': None}}, res)
def test_serialize_input_data(self):
res = sync_point.serialize_input_data({(3, 8): None})
self.assertEqual({'input_data': {u'tuple:(3, 8)': None}}, res)
def test_serialize_input_data(self):
res = sync_point.serialize_input_data({(3L, 8): None})
self.assertEqual({'input_data': [[[3L, 8], None]]}, res)
def check_resource(self, cnxt, resource_id, current_traversal, data,
is_update):
'''
Process a node in the dependency graph.
The node may be associated with either an update or a cleanup of its
associated resource.
'''
adopt_data = data.get('adopt_stack_data')
data = dict(sync_point.deserialize_input_data(data))
try:
cache_data = {in_data.get(
'name'): in_data for in_data in data.values()
if in_data is not None}
cache_data['adopt_stack_data'] = adopt_data
rsrc, stack = resource.Resource.load(cnxt, resource_id, is_update,
cache_data)
except (exception.ResourceNotFound, exception.NotFound):
return
tmpl = stack.t
if current_traversal != rsrc.stack.current_traversal:
LOG.debug('[%s] Traversal cancelled; stopping.', current_traversal)
return
current_deps = ([tuple(i), (tuple(j) if j is not None else None)]
for i, j in rsrc.stack.current_deps['edges'])
deps = dependencies.Dependencies(edges=current_deps)
graph = deps.graph()
if is_update:
if (rsrc.replaced_by is not None and
rsrc.current_template_id != tmpl.id):
return
try:
check_resource_update(rsrc, tmpl.id, data, self.engine_id)
except resource.UpdateReplace:
new_res_id = rsrc.make_replacement(tmpl.id)
LOG.info("Replacing resource with new id %s", new_res_id)
data = sync_point.serialize_input_data(data)
self._rpc_client.check_resource(cnxt,
new_res_id,
current_traversal,
data, is_update)
return
except resource.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, resource_id):
self._rpc_client.check_resource(cnxt,
resource_id,
current_traversal,
data, is_update)
return
except exception.ResourceFailure as ex:
reason = 'Resource %s failed: %s' % (stack.action,
six.text_type(ex))
self._handle_resource_failure(
cnxt, stack.id, current_traversal, reason)
return
input_data = construct_input_data(rsrc)
else:
try:
check_resource_cleanup(rsrc, tmpl.id, data, self.engine_id)
except resource.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, resource_id):
self._rpc_client.check_resource(cnxt,
resource_id,
current_traversal,
data, is_update)
return
except exception.ResourceFailure as ex:
reason = 'Resource %s failed: %s' % (stack.action,
six.text_type(ex))
self._handle_resource_failure(
cnxt, stack.id, current_traversal, reason)
return
graph_key = (resource_id, is_update)
if graph_key not in graph and rsrc.replaces is not None:
# If we are a replacement, impersonate the replaced resource for
# the purposes of calculating whether subsequent resources are
# ready, since everybody has to work from the same version of the
# graph. Our real resource ID is sent in the input_data, so the
# dependencies will get updated to point to this resource in time
# for the next traversal.
graph_key = (rsrc.replaces, is_update)
try:
for req, fwd in deps.required_by(graph_key):
propagate_check_resource(
cnxt, self._rpc_client, req, current_traversal,
set(graph[(req, fwd)]), graph_key,
input_data if fwd else None, fwd)
check_stack_complete(cnxt, rsrc.stack, current_traversal,
resource_id, deps, is_update)
except sync_point.SyncPointNotFound:
# Reload the stack to determine the current traversal, and check
# the SyncPoint for the current node to determine if it is ready.
# If it is, then retrigger the current node with the appropriate
# data for the latest traversal.
stack = parser.Stack.load(cnxt, stack_id=rsrc.stack.id)
if current_traversal == rsrc.stack.current_traversal:
LOG.debug('[%s] Traversal sync point missing.',
current_traversal)
return
current_traversal = stack.current_traversal
current_deps = ([tuple(i), (tuple(j) if j is not None else None)]
for i, j in stack.current_deps['edges'])
deps = dependencies.Dependencies(edges=current_deps)
key = sync_point.make_key(resource_id, current_traversal,
is_update)
predecessors = deps.graph()[key]
def do_check(target_key, data):
self.check_resource(resource_id, current_traversal,
data)
try:
sync_point.sync(cnxt, resource_id, current_traversal,
is_update, do_check, predecessors, {key: None})
except sync_point.sync_points.NotFound:
pass
